diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2644a6d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.star linguist-language=Starlark + +# ignore YAML files generated by kubebuilder +pkg/internal/tests/cluster/crd/** linguist-generated=true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..e5df4fa --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,59 @@ +name: CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + lint-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - "1.22" + env: + GOPRIVATE: github.com/reddit/achilles-sdk-api + + container: + image: golang:${{ matrix.go-version }} + + steps: + - uses: actions/checkout@v3 + + - name: Workaround Git Security Warning + run: | + # Workaround a bug in github actions: + # https://github.com/actions/runner-images/issues/6775. + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + known_hosts: ${{ secrets.KNOWN_HOSTS }} + key: ${{ secrets.GO_MODULE_PRIVATE_KEY }} + + - name: Setup access for private Go modules + run: | + git config --global url."ssh://git@github.com/".insteadOf https://github.com/ + + - name: Generate + run: | + # needed for running `tar -xJv` for installing shellcheck + apt-get update + apt-get install xz-utils + + make generate + git status + git diff + test -z "$(git status --porcelain)" + + # lint code + make lint + git status + git diff + test -z "$(git status --porcelain)" + + - name: Test + run: make test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f6c781b --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin +testbin/* + +# Test binary, docker with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Kubernetes Generated files - skip generated files, except for vendored files +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ +tilt_modules/ + +*.kubeconfig + +# local development tilt settings +tilt_config.json + +# goreleaser +dist/ + +# terraform +.terraform/ diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..517bd1c --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @reddit/achilles diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0949b66 --- /dev/null +++ b/Makefile @@ -0,0 +1,63 @@ +SHELL:=/bin/bash + +PWD := $(PWD) +CONTROLLER_GEN := $(PWD)/bin/controller-gen +CONTROLLER_GEN_CMD := $(CONTROLLER_GEN) +GOSIMPORTS := $(PWD)/bin/gosimports +GOSIMPORTS_CMD := $(GOSIMPORTS) +STATICCHECK := $(PWD)/bin/staticcheck +STATICCHECK_CMD := $(STATICCHECK) + +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.29 +ENVTEST := $(PWD)/bin/setup-envtest +ENVTEST_CMD := $(ENVTEST) + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go install -modfile=tools/go.mod $(2) ;\ +} +endef + +.PHONY: test-manifests +test-manifests: $(CONTROLLER_GEN) + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./pkg/internal/tests/api/..." output:crd:artifacts:config=pkg/internal/tests/cluster/crd/bases + $(CONTROLLER_GEN) object paths="./pkg/internal/tests/api/..." + +.PHONY: generate +generate: test-manifests $(GOSIMPORTS) + go generate ./... + $(GOSIMPORTS_CMD) -local github.com/reddit/achilles-sdk -l -w . + +KUBEBUILDER_ASSETS = $(shell $(ENVTEST_CMD) --arch=amd64 use $(ENVTEST_K8S_VERSION) -p path) +.PHONY: test +test: $(ENVTEST) test-manifests + KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race ./... + +.PHONY: lint +lint: $(STATICCHECK) $(GOSIMPORTS) + cd tools && go mod tidy + go mod tidy + go fmt ./... + go list ./... | grep -v encoding/json | xargs go vet # ignore forked encoding/json pkg + go list ./... | grep -v encoding/json | xargs $(STATICCHECK_CMD) # ignore forked encoding/json pkg + $(GOSIMPORTS_CMD) -local github.com/reddit/achilles-sdk -l -w . + +$(CONTROLLER_GEN): + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen) + +$(KUSTOMIZE): + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4) + +$(GOSIMPORTS): + $(call go-get-tool,$(GOSIMPORTS),github.com/rinchsan/gosimports/cmd/gosimports) + +$(STATICCHECK): + $(call go-get-tool,$(STATICCHECK),honnef.co/go/tools/cmd/staticcheck) + +$(ENVTEST): + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest) diff --git a/README.md b/README.md new file mode 100644 index 0000000..c2bc477 --- /dev/null +++ b/README.md @@ -0,0 +1,88 @@ +# Achilles SDK +[![Reticle Badge](https://reticle.snooguts.net/api/reticle_badge?repo_name=reddit/achilles-sdk)](https://reticle.snooguts.net/repos/reddit/achilles-sdk) + +> Achilles home: [go/achilles](https://go.snooguts.net/achilles) + +The Achilles SDK offers efficient +[controller](https://kubernetes.io/docs/concepts/architecture/controller/) and +[operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) +creation by allowing engineers to focus on defining their automation business logic, modeled as transitions between resources states (i.e. an FSM). This +significantly lessens the knowledge of controller and Kubernetes plumbing +typically required to build controllers. It also forces standardization to +ensure consistency amongst controllers at Reddit, providing common functionality +such as metrics, logging, reusable API structs, etc.. + +As an example, assume you wanted to create a declarative API that allows +developers to request object storage for their application. The API you wish to +expose may look something like: + +```yaml +apiVersion: infra.snooguts.net/v1alpha1 +kind: ObjectStorage +metadata: + name: persistent-storage + namespace: web-apps +spec: + encrypted: true + region: us-east + versioning: true +``` + +To define this API, you'll use a common-convention with Go structs to +automatically generate the +[CustomResourceDefinition](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#:~:text=The%20CustomResourceDefinition%20API%20resource%20allows,storage%20of%20your%20custom%20resource.) +(API). Then you'll define the states and transition logic, such as: + +![FSM Flow](docs/imgs/fsm-flow.png) + +Along with doing the underlying plumbing to support this controller, +achilles-sdk is providing you logging, metrics, rate-limiting, and +more. + +Lastly, in modeling your controller logic this way, you end up with a clean +directed graph that models a +[finite-state-machine](https://en.wikipedia.org/wiki/Finite-state_machine) +(FSM). This can significantly reduce the cognitive overhead of reasoning about +what a controller is doing or the state an object is in. More on FSM in the +documentation. + +## Documentation + +* [Creating a controller](https://pages.github.snooguts.net/reddit/achilles-docs/dev/sdk/tutorial/) + * Comprehensive guide on the implementation of a controller with + achilles-sdk. +* [Reconciler (FSM) Framework](docs/README.md) + * Overview of how achilles-sdk works by offering a finite-state machine + orchestrated with a Kubernetes reconciler. + +## How to Contribute +1. Clone the repo locally. +2. Create a new branch + - Make sure the branch is pushed to this repo, rather than from a fork. This allows us to easily test the branch code in our Tilt environment. +3. Make changes and test. +4. Submit a Pull Request with an appropriate description. +5. Merge PR once it is approved. + +## Releasing +After incorporating your changes into the achilles-sdk repository, you can publish a new release to make the updated functionality available for repos that consume achilles-sdk such as [achilles](https://github.snooguts.net/reddit/achilles). + +#### Publish a new release: +1. Navigate to the [GHE releases page for achilles-sdk](https://github.com/reddit/achilles-sdk/releases) and click the “Draft a new release” button in the top right +2. Choose the appropriate next semantic version (`major.minor.patch`): + - If there are breaking code changes (e.g., new function signatures for public functions), increment the minor version if the release is pre-1.0, otherwise increment the major version + - If there are no breaking changes, increment the patch version if pre-1.0, otherwise increment the minor version if adding functionality or the patch version if fixing bugs + - To create a tag with the new version, click the “Choose a tag” button. Enter the new version number and click “Create new tag: `$your-tagname-here` on publish.” +3. Click the “Generate release notes” button and verify that the changes align with expectations. If everything looks good, publish the release + +#### Make updated functionality available for repos like achilles: +1. Create a branch in the achilles repository +2. Run the following command to upgrade the achilles-sdk version (replace v0.7.1 with the new version number): + `go get github.com/reddit/achilles-sdk@v0.7.1` +3. Run a `git diff` and validate that the changes made to `go.mod` and `go.sum` are as expected. The diff should look something like the changes in this [PR](https://github.snooguts.net/reddit/achilles/pull/1135/files). +4. Create a PR in the achilles repo with the newly upgraded achilles-sdk version + +Note: There are several other repos that also consume achilles-sdk but the current pattern is to allow consumers to update at their own leisure. The main repo that should for sure be updated to adopt the latest features is `reddit/achilles`. + +## Questions + +If you have any questions, please reach out to the `#achilles` Slack channel. diff --git a/docs/envtest.md b/docs/envtest.md new file mode 100644 index 0000000..d8e1656 --- /dev/null +++ b/docs/envtest.md @@ -0,0 +1,305 @@ +# envtest + +This document describes what `envtest` integration tests are, why they are valuable for testing Kubernetes controllers, +and how to write them. + +## What is `envtest`? + +`envtest` is a binary that runs an instance of `kube-apiserver` and `etcd`, enabling integration testing of Kubernetes +controllers and its interaction against the Kubernetes control plane. It is part of the [Kubebuilder](https://book.kubebuilder.io/) project, +and the `envtest` docs can be [found here](https://book.kubebuilder.io/reference/envtest.html). + +The Achilles SDK provides a Go wrapper around `envtest`, making it easy to programmatically start, stop, and integrate controllers being tested against +the test environment. + +## Why Use `envtest`? + +The bulk of the implementation of any Kubernetes controller, regardless of business logic, reduces down to CRUD operations +against the Kubernetes API server. Therefore, a controller's core correctness can be exercised as asserting that the +controller instantiates the expected Kubernetes state (i.e. child objects) in response to changes in the declared state (i.e. parent objects). + +Furthermore, the Kubernetes API server imposes a number of constraints that only surface at runtime when the client (i.e. the controller) sends the request to the server. + +Examples of these constraints include: + +1. RBAC control: Is your controller configured with the correct Kubernetes RBAC for CRUDing the resources it needs to? +2. Core API semantics + 1. Does your controller respect [Kubernetes naming constraints](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/)? + 2. Does your controller send updates correctly? For example, does it specify the [`metadata.resourceVersion`](https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions) of the objects it reads and writes? +3. Update semantics: Is your controller serializing Go structs to YAML correctly? + +Lastly, Kubernetes controllers are dynamic processes that communicate asynchronously with the kube-apiserver. The actual +state instantiated by the controller and the kube-apiserver are _eventually consistent_, meaning that the actual state +eventually converges to the desired state. It is difficult to faithfully reproduce an eventually consistent environment +without running these asynchronous processes in your test environment. + +## How to write `envtest` integration tests + +### Exercise a Single Controller Per Test + +We recommend writing a single `envtest` integration test for each controller you write. This test should only exercise +a single controller's logic, and should not test the interaction between multiple controllers. + +If your controller interacts with other controllers or other APIs, your test should _mock_ the behavior of those external +systems, similar to classical unit testing methodology. For instance, if your controller creates a Deployment object +and waits until that Deployment becomes ready, you would insert test logic that emulates the Kubernetes control plane +processing the Deployment and setting its status to `Ready` or `Failed`. + +### Setting Up Your Test + +We use the [Ginkgo](https://onsi.github.io/ginkgo/) testing framework to write our tests. Ginkgo is a BDD-style testing +framework that allows us to write tests in a behavior-driven format. + +We use the [Gomega](https://onsi.github.io/gomega/) matcher library to write assertions in our tests. It's especially +useful for ergonomically making asynchronous assertions, required in eventually consistent systems. In most cases you'll +use the `Eventually` matcher to assert that a condition will eventually be true, and the `Consistently` +matcher to assert that a condition will remain true for a period of time. + +To get started, install the `ginkgo` CLI: + +```shell +go install github.com/onsi/ginkgo/v2/ginkgo@latest +``` + +If ginkgo is already installed, make sure you are running ginkgo v2. If you are running v1, upgrade to ginkgo v2 by following [this guide](https://github.com/onsi/ginkgo/blob/ver2/docs/MIGRATING_TO_V2.md#upgrading-to-ginkgo-20). + +Next, create a new test suite by running: + +```shell +cd /path/to/your/controller/package +ginkgo bootstrap . +``` + +This will create a `*_suite_test.go` file, which will contain the setup for the test suite. +Inside of this file, we scaffold a new `envtest` IT with the following Go code: + +```golang +package mycontroller_test // we recommend using a different package than your controller package so your test exercises only the public interface of the controller package + +import ( + "context" + "path" + "path/filepath" + "testing" + "time" + + "github.com/fgrosse/zaptest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.snooguts.net/reddit/achilles-sdk/pkg/fsm/metrics" + "github.snooguts.net/reddit/achilles-sdk/pkg/logging" + libratelimiter "github.snooguts.net/reddit/achilles-sdk/pkg/ratelimiter" + "github.snooguts.net/reddit/achilles-sdk/pkg/test" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + ctrlzap "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.snooguts.net/reddit/mycontroller/internal/controllers/mycontroller" + "github.snooguts.net/reddit/mycontroller/internal/controlplane" + libtest "github.snooguts.net/reddit/mycontroller/internal/test" + ctrlscheme "github.snooguts.net/reddit/mycontroller/pkg/scheme" +) + +var ( + ctx context.Context + scheme *runtime.Scheme + log *zap.SugaredLogger + testEnv *test.TestEnv + testClient client.Client +) + +func TestMyController(t *testing.T) { + RegisterFailHandler(Fail) + + log = zaptest.LoggerWriter(GinkgoWriter).Sugar() // wires up the controller output to the Ginkgo test runner so logs show up in the shell output + ctrllog.SetLogger(ctrlzap.New(ctrlzap.WriteTo(GinkgoWriter), ctrlzap.UseDevMode(true))) + + RunSpecs(t, "mycontroller Suite") +} + +var _ = BeforeSuite(func() { + // default timeouts for "eventually" and "consistently" Ginkgo matchers + SetDefaultEventuallyTimeout(40 * time.Second) + SetDefaultEventuallyPollingInterval(200 * time.Millisecond) + SetDefaultConsistentlyDuration(3 * time.Second) + SetDefaultConsistentlyPollingInterval(200 * time.Millisecond) + + ctx = logging.NewContext(context.Background(), log) + rl := libratelimiter.NewDefaultProviderRateLimiter(libratelimiter.DefaultProviderRPS) + + // add test CRD schemes + scheme = ctrlscheme.MustNewScheme() + + // fetch external CRDs. + // TODO: this is optional. You only need this if your controller makes use of + // other codebases' resources. + externalCRDDirectories, err := test.ExternalCRDDirectoryPaths(map[string][]string{ + "github.com/some/other/repo/apis/v1alpha1": { + path.Join("config", "crd", "bases"), + }, + }, libtest.RootDir()) + Expect(err).ToNot(HaveOccurred()) + + testEnv, err = test.NewEnvTestBuilder(ctx). + WithCRDDirectoryPaths( + append(externalCRDDirectories, + // enumerate all directories containing CRD YAMLs + filepath.Join(libtest.RootDir(), "manifests", "base", "crd", "bases"), + )). + WithScheme(scheme). + WithLog(log.Desugar()). + WithManagerSetupFns( + func(mgr manager.Manager) error { + return mycontroller.SetupController( + ctx, + controlplane.Context{ + Metrics: metrics.MustMakeMetrics(scheme, prometheus.NewRegistry()), + }, + mgr, + rl, + ) // wires up controller being tested to the kube-apiserver + }, + ). + WithKubeConfigFile("./"). // test suite will output a kubeconfig file located in the specified directory + Start() // start invokes the `envtest` binary to start the `kube-apiserver` and `etcd` processes on the host + Expect(err).ToNot(HaveOccurred()) + testClient = testEnv.Client +}) + +// AfterSuite tears down the test environment by terminating the `envtest` processes once the test finishes +// Without this, the host will have orphaned `kube-apiserver` and `etcd` processes that will require manual cleanup +var _ = AfterSuite(func() { + By("tearing down the test environment", func() { + if testEnv != nil { + Expect(testEnv.Stop()).To(Succeed()) + } + }) +}) + +``` + +Now, you can set up envtests in `_test.go` files, just like other Go tests. Here's an example: + +```golang +package mycontroller_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// TODO: Put your env tests here. +var _ = Describe("mycontroller", Ordered, func() { + It("should work", func() { + Eventually(func(g Gomega) { + g.Expect(true).To(Equal(true)) + }).Should(Succeed()) + }) +}) + +``` + +When executing this test via `go test`, the `envtest` binary will start the `kube-apiserver` and `etcd` processes on the host. +The controller being tested will be wired up to the `kube-apiserver` and will be able to interact with the Kubernetes control plane. + +### Writing Your Test + +`envtest` ITs should be expressed in a behavioral manner, which is higher level than how you might express a unit test. +For controller automation, the general structure of a test would be to mimic how an actor would use the custom resource: + +1. Actor (human or program) creates the custom resource +2. Test asserts that the controller processes the custom resource and performs the expected actions: + 1. Creates expected child resources + 2. Updates the custom resource's status + 3. Performs expected actions against other integrated external systems (e.g. issues a request to a REST API) +3. Actor updates the custom resource +4. Test asserts that the controller processes the update and performs the expected actions +5. Actor deletes the custom resource +6. Test asserts that the controller performs expected cleanup actions + +Here is an example of a test that exercises the controller's behavior when a custom resource is created: + +```golang +var _ = Describe("MyController", func() { + Context("when a MyResource is created", func() { + It("should create a MyChildResource", func() { + By("creating a MyResource", func() { + myResource := &mycontroller.MyResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-resource", + Namespace: "default", + }, + Spec: mycontroller.MyResourceSpec{ + // fill in the spec fields + }, + } + Expect(orchClient.Create(ctx, myResource)).To(Succeed()) + }) + + By("waiting for the MyChildResource to be created", func() { + expected := &mycontroller.MyChildResource{ + // fill in expected state + } + + Eventually(func(g Gomega) { + actual := &mycontroller.MyChildResource{} + g.Expect(orchClient.Get(ctx, client.ObjectKey{Name: "my-child-resource", Namespace: "default"}, actual)).To(Succeed()) + g.Expect(actual).To(Equal(expected)) + }).Should(Succeed()) + }) + }) + }) +}) +``` + +Notice that we use an "Eventually" assertion. Because Kubernetes is an eventually consistent system, the test assertion +must poll for the expected state, with a user-configured timeout and polling interval. + +`envtest` ITs can also exercise controller failures modes by emulating conditions under which your controller will error +out. + +To see a full example of a `envtest` IT, refer to the [Achilles Federation controller test](https://github.snooguts.net/reddit/achilles/blob/f5f453b25216fd25f68c22b453345eb6777efbf0/orchestration-controller-manager/internal/controllers/federation/federation_reconciler_test.go). + +### Running Your Test + +To run your test, execute the following command: + +```shell +go test ./path/to/your/controller/package +``` + +### Debugging Your Test + +Controllers are more difficult to debug that single-threaded in-memory tests, both because of the asynchronous nature of +the controller and control plane, and because `envtest` runs the control plane as an out-of-band process (rather than embedded +in the test process). + +That being said, we have some extremely helpful methods for debugging controllers. + +#### Manually Interact with the Envtest Control Plane + +By calling `.WithKubeConfigFile(/path/to/kubeconfig)` in your envtest builder, the test suite will output a kubeconfig file that you can +use to interact with the control plane using `kubectl`. As long as the test suite is still running, the envtest control plane +will be accessible via `kubectl`. + +The recommended workflow is to execute the test in a debugger and pause the test execution at the point where you want to +inspect the control plane. Then, pass the kubeconfig to `kubectl` to inspect the state of the control plane: + +```shell +kubectl --kubeconfig /path/to/kubeconfig get pods -A +``` + +This allows the test author to inspect arbitrary state on the control plane, which is much more efficient than +instrumenting the test logic or controller logic to output all relevant state. + +#### Run the Tests in a Debugger + +Running tests in an interactive debugger allows you to pause the test execution and inspect the _in-memory_ state +of the controller, the control plane, and the test environment. + +If using Jetbrains GoLand, you can set breakpoints in your test code and run the test with the debugger. The test will pause +at the breakpoint, and you can inspect the state of the controller, the control plane, and the test environment. diff --git a/docs/sdk-apply-objects.md b/docs/sdk-apply-objects.md new file mode 100644 index 0000000..1622bc0 --- /dev/null +++ b/docs/sdk-apply-objects.md @@ -0,0 +1,307 @@ +# Applying Objects + +This document describes conventions and patterns around performing updates to Kubernetes resources. + +## OutputSet + +The `achilles-sdk`'s FSM reconciler supplies an [`OutputSet` abstraction](https://github.snooguts.net/reddit/achilles-sdk/blob/340f21a1aa4595651c8627fe754a44082bfab34b/pkg/fsm/types/output.go#L17) +that should satisfy _most_ use resource update use cases. + +The following illustrates the typical pattern: + +```golang +var state = &state{ + Name: "some-state", + Condition: SomeCondition, + Transition: func( + ctx context.Context, + r types.ReconcilerContext, + ctrlCtx controlplane.Context, + object *v1alpha1.SomeObject, + out *sets.OutputSet, + ) (*state, types.Result) { + // perform some logic and build output objects + appProject := &argov1alpha1.AppProject{ + ObjectMeta: ctrl.ObjectMeta{ + Name: AppProjectName(redditWorkload.Name, redditWorkload.Namespace), + Namespace: ctrlCtx.ArgoCDNamespace, + }, + Spec: argov1alpha1.AppProjectSpec{ + SourceRepos: []string{ + oci.HelmProdRegistryURL, + oci.HelmDevRegistryURL, + }, + SourceNamespaces: []string{redditWorkload.Namespace}, + Destinations: []argov1alpha1.ApplicationDestination{ + {Name: "*", Namespace: redditWorkload.Namespace}, + }, + }, + } + + // apply output objects via output set + out.Apply(appProject) + + return nextState, types.DoneResult() + }, +} +``` + +Our hypothetical controller builds an ArgoCD AppProject. The developer then simply passes +them into the `OutputSet` and the achilles-sdk handles applying those resources and their declared state to the Kubernetes +server. + +If the output object does not exist, it is created. If the output object already exists, it is updated to match the +configuration (metadata, spec, and status) constructed by your controller. + +The update only includes fields specified by the in-memory copy of the object. If using the SDK's `OutputSet` or `ClientApplicator.Apply(...)` abstractions, +we strongly recommend that you build the object from scratch rather than mutate a copy read from the server. This ensures +that you only update fields that you intend to update (and don't accidentally send data that you don't intend on having your +logic update). More advanced use cases may require a different "mutation-based" approach, discussed more below. + +Critically, on updates, only the fields specified by your code are updated on the server. **If you omit fields +whose values are pointer types, maps, or slices, they will not be updated on the server.** More detailed discussion on +apply semantics in below. + + +## Detailed Apply Semantics + +### Data Serialization + +There's a surprising amount of complexity in how clients send resource updates to the kube-apiserver. +Controllers are clients of the Kubernetes API Server. They send CRUD requests to the server to control Kubernetes +configuration. `Create` and `Update` request bodies contain the desired object state. These request bodies +are serialized from the underlying in-memory type to YAML (or JSON) before being sent to the server. + +For `Patch` update requests, only the fields that appear in the request body (which is determined by the underyling +implementation's serialization behavior) are updated. + +As discussed below, the Achilles SDK models updates in a manner that facilitates multiple actors gracefully managing +mutually exclusive fields on the same objects. This is achieved by controllers selectively serializing only the fields +they manage, and omitting fields they do not manage. + +### Achilles SDK Conventions + +Achilles controllers (and more generally, all Kubernetes controllers) should honor the following assumptions. Violation +of these assumptions leads to interoperability issues and can cause controllers to overwrite each other's updates. + +**Assumption 1: One Owner Per Field** + +For all Kubernetes objects that our controllers read and/or write, we must make the assumption that +every **field** in `metadata`, `spec`, and `status` has a single owner. "Owner" in this context refers to a program +or human that mutates the field. This constraint is not bespoke to the Achilles SDK; it is a general recommendation +for all Kubernetes controllers. The kube-apiserver can even enforce this constraint through a feature called +["server-side-apply"](https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management). We don't +currently use SSA in the Achilles SDK, but it is a feature that can be integrated in the future. For now, the onus of +following this convention rests on the controllers that manage a given resource. + +Violation of this assumption leads to conflicting updates where two owners fight over the same field, which can lead to +a malfunctioning system. + +**Assumption 2: For objects with multiple owners, all fields are pointer types, maps, or slices with the `omitempty` JSON tag** + +First, a quick primer on how Go serializes structs to JSON or YAML using the `encoding/json` package: + +1. Fields that are scalar types (int, string, bool, etc.) are always serialized to their JSON or YAML representation + i. There is no way to omit these fields from the serialized output. +2. Fields that are pointer types, maps, or slices and marked with the `omitempty` JSON tag[^1] are serialized to their JSON or YAML representation _only if the field is non-nil_ + i. If the field is nil, it is omitted from the serialized output. + ii. If the field is empty but non-nil (e.g. `[]` for slices, `{}` for maps and structs), it is serialized to its empty JSON representation (`[]` and `{}` respectively). + +Given these serialization mechanics, for objects that have multiple owners acting on mutually exclusive fields, we must ensure that +all `spec` and `status` fields are types that allow omission when serializing from Go types (pointer types, maps, or slices). +This means that actors can send updates while _omitting_ fields that they do not own, thus preventing collisions +with other owners. + +Following these two assumptions, we can optimistically apply all object updates without utilizing Kubernetes' [resource version](https://github.snooguts.net/reddit/reddit-helm-charts#versioning) +because there is no risk that any actor's update will conflict with or overwrite that of a different actor's. + +We also update all objects using [JSON merge patch semantics](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/#use-a-json-merge-patch-to-update-a-deployment), +([RFC here](https://tools.ietf.org/html/rfc7386)), which offers the simplest mental model for merging an update +against existing object state. + +**Deleting Fields** + +**zero value equivalent to field omission:** + +This applies in APIs where a field's zero value (which depends on the type) is semantically equivalent to the field +being absent. In these cases, setting the field to the zero value is equivalent to deleting the field. + +Using JSON merge patch semantics, deleting a field requires that the request body contain the field with an empty value +(`0` for numerics, `""` for strings, `false` for bools, `[]` for slices and `{}` for maps or structs). + +Assuming your field is a pointer type, map, or slice with the `omitempty` JSON tag, you can delete a field by setting it +to the empty but non-nil value for the given type. + +**zero value distinct from field omission:** + +In APIs where a field's zero value is not semantically equivalent to the field being absent, deleting the field requires +using an `Update` operation (rather than a `Patch` operation) to overwrite the entire object (same approach +as described below under "Deleting key-value pairs from map types"). + +## Advanced Apply Patterns + +The two assumptions above do not always hold, especially when using 3rd party CRDs whose types you do not control, +that are implemented by controllers whose behavior you don't control. + +Use the following workarounds when presented with these convention departures. + +**Kubernetes Resource Lock** + +Kubernetes CRDs are supplied with a resource lock, tracked by the `metadata.resourceVersion` field. +This field is an integer that represents the latest version of the resource that the server has persisted. +If an update or patch request specifies this field, the kube-apiserver will reject the request if the request's field does not match +the server's field. This guarantees that the client is operating on the latest version of the object. +Full details on the design and implementation [can be found here](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency). + +When would you want to use this feature? + +The backing Kubernetes client cache supplied by controller-runtime [can serve stale (i.e. outdated) data](https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md#q-my-cache-might-be-stale-if-i-read-from-a-cache-how-should-i-deal-with-that). +If your logic reads data from the kube-apiserver and then uses that data to update the object, you may inadvertently +update the object with stale data. If sending stale data has negative side effects, you should use the resource lock +to guarantee that stale updates are never sent. + +For cases where an external controller is managing some fields in the `spec`, whose types _must_ be serialized (meaning that +you cannot avoid having your controller send some value for fields it doesn't manage because you don't control +the API struct definition and because the field is not a pointer type, map, or slice with the `omitempty` JSON tag), +the workaround is for your controller to perform a "read-modify-write" operation while using the Kubernetes resource lock, +which consists of: + +1. read the object from the server +2. modify only the fields that your controller manages +3. write the object back to the server while using the resource lock + +Usage of the resource lock (i.e. sending the update request with `metadata.resourceVersion` populated) ensures that +the Kubernetes server will reject the update if it's operating on a version of the object that is out of date. +This guarantees that your controller will not overwrite data managed by other controllers. + +To use the resource lock, do the following: + +```golang +import "github.snooguts.net/reddit/achilles-sdk/pkg/io" + +out.Apply(obj, io.WithResourceLock()) +``` + +**Deleting key-value pairs from map types** + +Since we're using JSON merge patch semantics by default, the only way to delete a KV pair from fields whose Go type is a +map is to overwrite the entire map.[^2] + +To perform a full object update, supply to `AsUpdate()` apply option like so: + + +```golang +import "github.snooguts.net/reddit/achilles-sdk/pkg/io" + +out.Apply(obj, io.AsUpdate()) +``` + +If this is a 3rd party CRD, you will likely need to pair the usage of `AsUpdate()` with `WithResourceLock()` to avoid +overwriting fields your controller does not manage. + +**Custom Management of Owner References** + +By default, the FSM reconciler adds an owner reference to all managed resources that links back to the reconciled object. +This enables Kubernetes-native garbage collection, whereby all managed resources will be deleted when the reconciled object +gets deleted. This default behavior makes sense in _most_ controller use cases. + +If your controller is intentionally managing owner references, you must disable this feature by using the `io.WithoutOwnerRefs()` +([link](https://github.snooguts.net/reddit/achilles-sdk/blob/dea76bcf6143aebce5ba0763f99c9f282b5b3415/pkg/io/options.go#L44)) +apply option. + +### Mutation Based Updates + +The Achilles SDK's `ClientApplicator` and `OutputSet` assume a "declarative-style" pattern where the update data is +built from scratch rather than mutated from a copy read from the server. However, there is a scenario where a mutation-based +approach is desirable—you want to minimize the performance cost of your controller updating a field with "stale" (i.e. outdated) data. + +The backing Kubernetes client cache supplied by controller-runtime [can serve stale (i.e. outdated) data](https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md#q-my-cache-might-be-stale-if-i-read-from-a-cache-how-should-i-deal-with-that), +which can lead to extra rounds of reconciliation before your controller converges the managed object into a steady (i.e. unchanging) state. + +Ideally your controller must implement idempotent reconciliations and should be tolerant of actuation even if they are +caused by stale data. But if performance becomes a concern, or you wish to optimize your controller to reduce the number +of reconciliations, you can use a mutation-based approach. + + +```golang + // Create or Update the deployment default/foo + deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + // NOTE: there's an analogous method for CreateOrPatch + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error { + // Deployment selector is immutable so we set this value only if + // a new object is going to be created + if deploy.ObjectMeta.CreationTimestamp.IsZero() { + deploy.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + } + } + + // update the Deployment pod template + deploy.Spec.Template = corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "busybox", + Image: "busybox", + }, + }, + }, + } + + return nil + }) + + if err != nil { + log.Error(err, "Deployment reconcile failed") + } else { + log.Info("Deployment successfully reconciled", "operation", op) + } +``` + +The implementation of `CreateOrUpdate` and `CreateOrPatch` is as follows: + +1. It performs a read from the server (fronted by the cache, so it can still be stale) +2. It executes the supplied mutation function to mutate the object +3. It sends the update or patch + +This implementation reduces the chance that stale data is sent to the server because of the initial read. But importantly, +it does not guarantee that stale data is not written. To have this guarantee, you must supply the Kubernetes resource lock +when sending your update (discussed above under "Kubernetes Resource Lock"). + +## FAQ + +1. What happens if my controller sends an update/patch request with stale data? + +Consider the following timeline: + +``` +t0: Controller A reads object from server +t1: Actor B updates object +t2: Controller A updates object with stale data +``` + +If the resource lock is used, the kube-apiserver will reject the update at `t2` because the resource version in the +update request does not match the server's resource version. + +If the resource lock is not used, the kube-apiserver will accept the update at `t2` and the object will be updated with +stale data. If Actor B responds by updating the object again, the object will be updated with Actor B's data, which will +in turn actuate Controller A. This cycle may repeat until Controller A eventually reads non-stale data and thus doesn't +overwrite Actor B. Cycles of this kind are essentially livelocks, which should be mitigated by using the resource lock. + +If Actor B does not react by updating the object again, then the object will remain in the state that Controller A set it to, +i.e. with stale data. There is no livelock in this scenario but the object is in an incorrect state. This can be mitigated +by using the resource lock. + +## References + +1. The full list of apply options lives under [`/pkg/io/options.go`](https://github.snooguts.net/reddit/achilles-sdk/blob/0a9d9df29d5201b0f3d689108547c3a97d819d82/pkg/io/options.go) +2. The client abstraction lives under [`/pkg/io/applicator.go`](https://github.snooguts.net/reddit/achilles-sdk/blob/0a9d9df29d5201b0f3d689108547c3a97d819d82/pkg/io/applicator.go) + +[^1]: Read more about [Go serialization here](https://pkg.go.dev/encoding/json#Marshal) +[^2]: We could theoretically implement or use a custom Go JSON marshaller that can output `key: null` to signal deletion of fields. diff --git a/docs/sdk-claim-claimed-reconcilers.md b/docs/sdk-claim-claimed-reconcilers.md new file mode 100644 index 0000000..b732ef0 --- /dev/null +++ b/docs/sdk-claim-claimed-reconcilers.md @@ -0,0 +1,35 @@ +# FSM Claim/Claimed Reconciler + +This guide walks you through the claim/claimed reconciler pattern available in the `achilles-sdk` and when to use it. + +## Background + +The `achilles-sdk` in addition to the base [FSM reconciler]({{< ref "dev/sdk/sdk-fsm-reconciler" >}}) also provides a claim/claimed reconciler pattern. +This pattern uses the same FSM semantics from a developer perspective. This pattern is built around two types of resources: +1. **Claim**: A claim resource is created by an end user or client claiming access to a resource. An example of this type of resource is [`RedisClusterClaim`](https://github.snooguts.net/reddit/achilles-redis-controllers/blob/main/api/storage/v1alpha1/redisclusterclaim_type.go) + which claims access to a `RedisCluster` instance. +2. **Claimed**: A claimed resource captures the actual instance of given object. An example of this type of resource is [`RedisCluster`](https://github.snooguts.net/reddit/achilles-redis-controllers/blob/main/api/storage/v1alpha1/rediscluster_type.go) + which captures the actual resources running in AWS. + +**The `claim` object is namespaced whereas the `claimed` object is cluster-scoped**. + + +## When to use the claim/claimed reconciler pattern? + +It's desirable for the `claim` object to be namespace scoped for one of the following reasons: + +1. The `claim` object is end-user facing and you want to ensure that users cannot mutate objects owned by other users by leveraging native Kubernetes RBAC, using the namespace as an isolation unit. +2. Your system organizes resources by namespace. For instance, all configuration pertaining to entity Y belongs in namespace Y. + 1. For example, Compute's `orchestration-controller-manager` controller provisions a namespace for each RedditCluster and places all resources pertaining to that RedditCluster in its own namespace. This allows easy inspection and cleanup of all state pertaining to a given RedditCluster. +3. You want to expose a namespace-scoped CR (the `claim` object) whose implementation (the `claimed` object) requires cluster-scoped resources. + 1. For instance, Storage's namespace-scoped `RedisClusterClaim` CR is implemented using cluster-scoped Crossplane managed resources. + 1. The `claimed` object being cluster scoped allows the `claim` object to manage and "own" both cluster-scoped resources like those exposed by Crossplane + and namespaced resources. Kubernetes explicitly disallows cross-namespace owner references ([ref](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/)). +4. You want to decouple the "request" (the `claim` object) from the "fulfillment" of that request (the `claimed` object), similar to Kubernetes' PVC and PV pattern. + 1. Note (April 17th, 2024) that the `achilles-sdk` hasn't implemented this decoupling pattern yet because no concrete use cases have required it. + +The `achilles-sdk` creates two independent reconcilers for the `claim` and `claimed` objects. +The `claim` reconciler is entirely managed by the sdk and does not require any work on the user's part ([ref](https://github.snooguts.net/reddit/achilles-sdk/blob/main/pkg/fsm/internal/reconciler_claim.go#L43)). +The `claim` reconciler is responsible for creating the `claimed` object if it does not exist and cascading a delete call when the `claim` object is deleted. +The developer is responsible for implementing the `claimed` reconciler using the exposed [FSM semantics]({{< ref "dev/sdk/sdk-fsm-reconciler" >}}). +As an example take a look at the [ASGRotator Controller](https://github.snooguts.net/reddit/achilles/blob/master/orchestration-controller-manager/internal/controllers/cloud-resources/asgrotator/controller.go). diff --git a/docs/sdk-finalizers.md b/docs/sdk-finalizers.md new file mode 100644 index 0000000..7cab77b --- /dev/null +++ b/docs/sdk-finalizers.md @@ -0,0 +1,39 @@ +# What are Finalizers? + +Finalizers is a feature in Kubernetes that prevent the deletion of an object until some conditions are met. Finalizers +alert controllers when an object is being deleted, allowing them to perform cleanup tasks before the object is removed. +More information about how finalizers work is available in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/). + +## Finalizers in the Achilles SDK + +The Achilles SDK has an optional feature to provide a `finalizerState` when creating a new controller. If a `finalizerState` is provided +the sdk automatically manages adding and removing the finalizer on the object being reconciled. The finalizer is added/updated on every reconcile +as long as the object is not being deleted. Once the object is being deleted (i.e. `metadata.DeletionTimestamp` is set), the sdk will call the +`finalizerState` provided by the controller. The finalizer will only be removed once the `finalizerState` returns a `types.DoneResult()`. +Until the finalizer is removed, the object will not be deleted and the sdk will call the `finalizerState` on every reconcile. + +Some examples of what the finalizerState can be used for are: +1. Cleaning up state in remote systems like Vault (e.g. deleting all managed Vault entities) +2. Deleting child Kubernetes objects in a particular order (i.e. deleting Crossplane InstanceProfiles before Roles due to a Crossplane limitation) + +Example to add a finalizer to an achilles controller: +```go +builder := fsm.NewBuilder( + ... +).WithFinalizerState(finalizerState) + +var finalizerState = &state{ + Name: "finalizer", + Condition: apicommon.Deleting(), + Transition: func(ctx context.Context, r types.ReconcilerContext, ctrlCtx controlplane.Context, secret *appv1alpha1.RedditSecret, out *types.OutputSet) (*state, types.Result) { + // cleanup logic + return nil, types.DoneResult() + }, +} + +``` +### Finalizer for claim/claimed reconciler + +For the claim/claimed reconciler, the sdk automatically adds a finalizer to the `claim` object. +This is to ensure the sdk is able to delete the `claimed` object when the `claim` object is deleted ([ref](https://github.snooguts.net/reddit/achilles-sdk/blob/main/pkg/fsm/internal/reconciler_claim.go)). +The behaviour of the finalizer on the `claimed` object is controlled by the `finalizerState` provided to the controller as explained above diff --git a/docs/sdk-fsm-reconciler.md b/docs/sdk-fsm-reconciler.md new file mode 100644 index 0000000..9b3aafa --- /dev/null +++ b/docs/sdk-fsm-reconciler.md @@ -0,0 +1,132 @@ +# FSM Reconciler + +This guide walks you through the FSM (finite state machine) controller framework, both the +programming mental model and common controller patterns with code examples. + +The goal of the FSM framework is to allow software engineers without extensive +experience building Kubernetes controllers to build correct and conventional declarative APIs, abstracting away +internal controller mechanics to allow the developer to focus on automation business logic. + +## Background + +For a brief conceptual overview of control-loop theory as it pertains to +Kubernetes, [see this document](https://kubernetes.io/docs/concepts/architecture/controller/). + +The FSM framework is built on top of [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime/), a +widely used SDK for building Kubernetes controllers. + +Controller-runtime reduces complexity of building controllers through simplifying opinions, the most important ones +being: + +- a controller is composed of two main parts, the trigger conditions and the reconciliation logic + - trigger conditions specify _when_ a control loop actuates and _which_ object gets operated upon + - reconciliation logic specifies _what_ operations to perform for the object being reconciled (henceforth referred + to as "parent object") +- each logical control loop is responsible for managing a single resource type (i.e. GVK) +- controller triggers are level-based, not edge-based + +Controller-runtime describe some of their opinions and +conventions [here](https://github.com/kubernetes-sigs/controller-runtime/blob/main/pkg/doc.go). + +The FSM framework extends controller-runtime with additional opinions and structure that further simplify the process +of building controllers. These will be discussed below. + +## Finite State Machine Model + +The FSM framework provides additional structure over +controller-runtime's [monolithic "Reconcile()" method](https://github.com/kubernetes-sigs/controller-runtime/blob/dca0be70fd22d5200f37d986ec83450a80295e59/pkg/reconcile/reconcile.go#L93) +by modeling reconciliation as a finite state machine. Furthermore, the finite state machine has the additional +constraint +that **it must be a directed acyclic graph**. Cycles _within a single reconciliation invocation_ are detected and reported as runtime errors. Crucially, this does not prevent the controller from continuously reconciling on a periodic interval (especially for use cases that require polling some upstream system). + +Additionally, each reconciliation starts from the FSM's initial state rather than starting from the last reached state. +This implies the following: + +- all paths through the FSM graph must be idempotent +- all FSM states must be reachable by observing persisted state available to the controller + +These design constraints ensure controller correctness by forcing the developer to write their reconciliation logic in +a manner that is both idempotent and dependent on externally persisted state, rather than state internal to the +controller, which can easily diverge from the actual state of the world. The resulting control loop logic is resilient +to controller restarts and any runtime errors. + +## States + +Every state in the FSM maps to a [status condition](https://maelvls.dev/kubernetes-conditions/) on the parent object, +which is defined by the +developer [for each state](https://github.snooguts.net/reddit/achilles/blob/c0ddc4dadb6a7613552598da773bd77b80b15c0c/lib/fsm/types/transitions.go#L52) +. If the state completes successfully, the status condition's `Status` field will be set to true. Otherwise, in the case +of a +requeue result or error, the status field will be set to false. + +The tracking of states via status condition adheres to Kubernetes API best practices by providing an externally +observable +signal to dependencies of the API. Other actors (programs or humans) can treat the status conditions of FSM-backed APIs +as an authoritative source of truth on its status. + +## Transitioning Between States + +Each state defines the next state to transition to the current state completes successfully. The next state can vary +based on logical conditions, allowing the expression of branching paths. + +Each state +defines [a result type](https://github.snooguts.net/reddit/achilles/blob/c0ddc4dadb6a7613552598da773bd77b80b15c0c/lib/fsm/types/transitions.go#L118-L165) +. +Broadly speaking, there are three types of results: + +1. **done**—the state has finished successfully, the reconciler can transition to the next state or, in the case of a + terminal state, simply complete +2. **requeue**—instructs the reconciler to trigger again after a user-specified amount of time. This is used in cases + where + a controller is waiting for an external condition to be fulfilled. +3. **error**—the reconciler logs an error message and will retrigger, the delay of which is subject to exponential + backoff + +A requeue result is typically used over an error result when the external condition is expected to be eventually +consistent, and +thus its retrigger should not be subject to exponential delay. + +An error result is used when an external condition is not expected to be false. + +## Writing and Updating Managed Resources + +The majority of controllers involve creating and updating Kubernetes objects, whether they are CRDs or native resources. +The FSM framework provides +an [output object set abstraction](https://github.snooguts.net/reddit/achilles/blob/c0ddc4dadb6a7613552598da773bd77b80b15c0c/lib/fsm/types/transitions.go#L41) +for ensuring outputs. It provides the following functionality: + +- output objects are tracked via the parent object's status +- output objects have their owner references updated with the parent object + - this provides free garbage collection (i.e. the child objects will be deleted if the parent object is deleted) via + native Kubernetes garbage collection + +## Finalizer States + +[Kubernetes finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) can be used +to ensure the execution of logic that triggers when an object is deleted. Objects with a finalizer will remain in a +terminating state, but not get deleted from Kubernetes state, until the finalizer is removed. + +The FSM provides a way to add a separate FSM triggered upon deletion of +objects, [see this example](https://github.snooguts.net/reddit/achilles/blob/36c3aa3bde5a2590f5d914918a8cefdf1ef953a7/lib/fsm/test/test_fsm_reconciler.go#L39) +. The FSM automatically manages the attachment and removal of the finalizer. The finalizer will only be removed if the +finalizer FSM terminates successfully. + +## Trigger Conditions + +The FSM exposes the same trigger conditions as controller-runtime. + +When building a new controller, +use `.Manages` ([source](https://github.snooguts.net/reddit/achilles/blob/e8f58f6d9a66ab799da21ae9eb1cdc373e56e2d2/lib/fsm/controller.go#L76)) +to specify the type of object that is being managed by the controller. Each controller can only manage a single object +type. + +The FSM automatically wires up triggers for all [managed resources](##Writing and Updating Managed Resources). + +Additional trigger conditions can be wired up for arbitrary events via +the [`.Watches` method](https://github.snooguts.net/reddit/achilles/blob/e8f58f6d9a66ab799da21ae9eb1cdc373e56e2d2/lib/fsm/controller.go#L109) +. + +## Example FSM Controllers + +See [this simple example](https://github.snooguts.net/reddit/achilles/blob/1499fc7d792c9d717572bea58e85ccb597245bb3/lib/fsm/test/test_fsm_reconciler.go) +for reference on how to implement an FSM controller. diff --git a/docs/sdk-metrics.md b/docs/sdk-metrics.md new file mode 100644 index 0000000..cbe70cc --- /dev/null +++ b/docs/sdk-metrics.md @@ -0,0 +1,122 @@ +# Metrics and Monitoring + +This guide describes the metrics provided by the SDK and how to use them to monitor the health and performance of your +system. + +## Controller-runtime Metrics + +The Achilles SDK +integrates [controller-runtime metrics](https://github.com/kubernetes-sigs/controller-runtime/blob/1ed345090869edc4bd94fe220386cb7fa5df745f/pkg/internal/controller/metrics/metrics.go). +Controller-runtime metrics provide foundational metrics for understanding the performance and health of your controller. + +These metrics can be viewed in +the ["Controller Runtime" Grafana dashboard](https://grafana.kubernetes.ue1.snooguts.net/d/Md5CPB44k/controller-runtime?orgId=1&refresh=30s&var-cluster=orch-1&var-prometheus=monitoring%2Finfrared-system&var-controller=All&var-webhook=All&from=1721981744523&to=1722003344523). + +## SDK Metrics + +The Achilles SDK provides additional metrics that leverage SDK conventions and structures to provide more detailed +insights into the health and performance of your controller. + +These metrics are displayed in the following Grafana dashboards: + +1. [Achilles Reconciler Metrics](https://grafana.kubernetes.ue1.snooguts.net/d/p_-RmaUVk/achilles-reconciler-metrics?orgId=1&from=1721960667563&to=1722003867564) + 1. Provides a high level overview of your controller and its custom resources +2. [Achilles Reconciler Detailed Metrics](https://grafana.kubernetes.ue1.snooguts.net/d/0gaENrwVk/achilles-reconciler-detailed-metrics?orgId=1&from=1721982323248&to=1722003923249) + 1. Provides a detailed overview of particular reconcile loops of your controller. + +### **`achilles_resource_readiness`** + +This metric is a gauge that maps to an Achilles object's status conditions. By default, the SDK instruments metrics for the +status condition of type "Ready". Users can instrument additional status conditions by declaring the following when +building their reconciler: + +```golang +WithReconcilerOptions( + fsmtypes.ReconcilerOptions[v1alpha1.Foobar, *v1alpha1.Foobar]{ + MetricsOptions: fsmtypes.MetricsOptions{ + ConditionTypes: []api.ConditionType{ + // user specifies custom status condition types here + MyCustomStatusCondition.Type, + }, + }, + }, +) +``` + +This metric is emitted for each Achilles object, allowing operators to monitor the readiness of each API object +in their system. + +The metric has the following labels. +```c +achilles_resource_readiness{ + group="app.infrared.reddit.com", // the Kubernetes group of the resource + version="v1alpha1", // the Kubernetes version of the resource + kind="FederatedRedditNamespace", // the Kubernetes kind of the resource + name="demo-namespace-1", // the name of the resource + namespace="", // the namespace of the resource (empty for cluster-scoped CRDs) + status="True", // the status condition's "Status" field + type="Ready", // the status condition's "Type" field +} 1 // value of 1 means a status condition of the labelled status and type exists, 0 if it doesn't exist +``` + +### **`achilles_trigger`** + +This metric is a counter that provides insight into the events triggering your controller's reconcilers. It allows operators to reason +about the frequency and types of events that are causing the controller to reconcile. This is typically examined when +looking to reduce the frequency of reconciliations or understand suspected extraneous reconciliations. + +For a given reconciler, it is emitted for each (triggering object, event type) pair. + +The "type" label indicates the type of triggering object: + +1. **"self"** triggers happen by nature of controller-runtime's reconciler model, where any event on the reconciled object +triggers a reconciliation. +2. **"relative"** triggers occur through events on related objects. Related object triggers are wired up +using the `.Watches()` [builder method](https://github.snooguts.net/reddit/achilles-sdk/blob/bd2f3522d9e38b513f3a0f206f9bb9b0532c8b50/pkg/fsm/controller.go#L136). +3. **"child"** triggers occur through events on managed child objects (objects whose `meta.ownerRef` refers to the reconciled object). Child triggers +are wired up using the `.Manages()` [builder method](https://github.snooguts.net/reddit/achilles-sdk/blob/bd2f3522d9e38b513f3a0f206f9bb9b0532c8b50/pkg/fsm/controller.go#L96) + +```c +achilles_trigger{ + controller="ASGRotatorClaim", // the name of the reconciler + group="component.infrared.reddit.com", // the Kubernetes group of the triggering object + version="v1alpha1", // the Kubernetes version of the triggering object + kind="ASGRotator", // the Kubernetes kind of the triggering object + event="create", // the event type, one of "create", "update", "delete" + reqName="asg-rotator-managed", // the name of the triggering object + reqNamespace="dpwfeni-test-usva-aws-1", // the namespace of the triggering object (empty for cluster-scoped objects) + type="relative", // the trigger type, one of "relative", "self", or "child" +} 13 // the number of observed trigger events +``` + +### **`achilles_object_suspended`** + +This metric is a gauge that indicates whether an object is suspended. It is emitted for each reconciled object. +This metric is useful for alerting on any long-suspended objects. + +```c +achilles_object_suspended{ + group="app.infrared.reddit.com", // the Kubernetes group of the reconciled object + version="v1alpha1", // the Kubernetes version of the reconciled object + kind="FederatedRedditNamespace", // the Kubernetes kind of the reconciled object + name="achilles-test-apps", // the name of the reconciled object + namespace="", // the namespace of the reconciled object (empty for cluster-scoped objects) +} 0 // value of 1 means the object is suspended, 0 if it is not +``` + +### **`achilles_state_duration_seconds`** + +This metric is a histogram that provides performance insight into the duration of each state in the FSM. It is emitted +for each (reconciler, state) pair. + +```c +achilles_state_duration_seconds_bucket{ + group="app.infrared.reddit.com", // the Kubernetes group of the reconciled object + version="v1alpha1", // the Kubernetes version of the reconciled object + kind="FederatedRedditNamespace", // the Kubernetes kind of the reconciled object + state="check-federation-refs", // the name of the FSM state + le="0.99", // the percentile of the histogram distribution +} 183 // the duration in milliseconds +``` + +The average durations are graphed over time in the [Achilles Detailed Reconciler Metrics dashboard](https://grafana.kubernetes.ue1.snooguts.net/d/0gaENrwVk/achilles-reconciler-detailed-metrics?orgId=1&from=1721983467755&to=1722005067755). diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..364c17d --- /dev/null +++ b/go.mod @@ -0,0 +1,83 @@ +module github.com/reddit/achilles-sdk + +go 1.22.5 + +toolchain go1.22.7 + +require ( + github.com/fgrosse/zaptest v1.1.0 + github.com/go-logr/zapr v1.3.0 + github.com/gobuffalo/flect v0.3.0 + github.com/google/go-cmp v0.6.0 + github.com/iancoleman/strcase v0.2.0 + github.com/onsi/ginkgo/v2 v2.17.1 + github.com/onsi/gomega v1.32.0 + github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_model v0.6.0 + github.com/reddit/achilles-sdk-api v1.1.0 + github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace + github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/sync v0.7.0 + golang.org/x/time v0.5.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.30.0 + k8s.io/client-go v0.29.3 + k8s.io/utils v0.0.0-20231127182322-b307cd553661 + sigs.k8s.io/controller-runtime v0.17.3 + sigs.k8s.io/yaml v1.4.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/common v0.49.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.29.3 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fb91ae5 --- /dev/null +++ b/go.sum @@ -0,0 +1,250 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fgrosse/zaptest v1.1.0 h1:sK9hP0/xBoNX5qfFo3KWFluDXfc809APomI1QXuYELA= +github.com/fgrosse/zaptest v1.1.0/go.mod h1:vMnRSul6kW7kIUXZgnZZcDwyTn8k49ODfAULL8nmL5w= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= +github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.49.0 h1:ToNTdK4zSnPVJmh698mGFkDor9wBI/iGaJy5dbH1EgI= +github.com/prometheus/common v0.49.0/go.mod h1:Kxm+EULxRbUkjGU6WFsQqo3ORzB4tyKvlWFOE9mB2sE= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/reddit/achilles-sdk-api v1.1.0 h1:sbQv/qH/kaJU4UoAK7trZo76iA5zlgRpcbqGLcSCaNM= +github.com/reddit/achilles-sdk-api v1.1.0/go.mod h1:tKV9nH5k3TM5MGomS28JRzVyZ+yeJgdS2c5qMZe7fuI= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace h1:9PNP1jnUjRhfmGMlkXHjYPishpcw4jpSt/V/xYY3FMA= +github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= +k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= +sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go new file mode 100644 index 0000000..fb4dcaa --- /dev/null +++ b/pkg/bootstrap/bootstrap.go @@ -0,0 +1,233 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/go-logr/zapr" + "github.com/spf13/pflag" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + zaputil "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/reddit/achilles-sdk/pkg/logging" +) + +const ( + errNoValidKubeContext = "kubeconfig context must be specified when not in cluster" + errKubeContextSetInCluster = "kubeconfig context can not be specified when in cluster" +) + +// Options for starting a custom controller +type Options struct { + // InCluster specifies whether the controller should use the in-cluster k8s client config. + InCluster bool + + // KubeContext is the context name to use for the controller's k8s client. + KubeContext string + + // KubeConfig specifies the explicit path of the controller's k8s client config. + KubeConfig string + + // MetricsAddr is the bind address for the metrics endpoint + MetricsAddr string + // HealthAddr is the bind address for the healthcheck endpoints + HealthAddr string + + // enables verbose mode + VerboseMode bool + + // enables dev logger (instead of prod logger) + // NOTE: DO NOT set this to true in prod, it will crash on DPanic + DevLogger bool + + // Maximum QPS to the kube-apiserver from this client + ClientQPS float32 + + // Maximum burst for throttle + ClientBurst int + + // SyncPeriod determines the minimum frequency at which controllers will perform a reconciliation. + // This is a global setting that applies to all controllers. Defaults to 10 hours. + // Issue tracking sync periods per controller: https://github.com/reddit/achilles-sdk/issues/171 + SyncPeriod time.Duration +} + +func (o *Options) AddToFlags(flags *pflag.FlagSet) { + // kubeconfig parameters + flags.BoolVar(&o.InCluster, "incluster", false, "Uses the in-cluster Kubeconfig. Exactly one of `incluster` or `kubecontext` must be set") + flags.StringVar(&o.KubeContext, "kubecontext", "", "Specifies the kubeconfig context. Exactly one of `incluster` and `kubecontext` must be set") + flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Specifies the location of kubeconfig. Defaults to standard lookup strategy") + + flags.StringVar(&o.MetricsAddr, "metrics-addr", ":8080", "Bind address for metrics endpoint") + flags.StringVar(&o.HealthAddr, "health-addr", ":8081", "Bind address for health endpoint") + + // logging parameters + flags.BoolVar(&o.VerboseMode, "verbose", true, "Enable verbose logging") + flags.BoolVar(&o.DevLogger, "dev-logging", true, "Enable dev-mode logging (human-readable logs)") + + // client request rate parameters + flags.Float32Var(&o.ClientQPS, "client-qps", 5.0, "Maximum QPS to the kube-apiserver from the controller's client") + flags.IntVar(&o.ClientBurst, "client-burst", 10, "Maximum request/s burst to the kube-apiserver from the controller's client") + + flags.DurationVar(&o.SyncPeriod, "sync-period", 10*time.Hour, "Minimum frequency at which all controllers will perform a reconciliation.") +} + +// StartFunc is a function for starting a controller manager +type StartFunc func( + ctx context.Context, + mgr manager.Manager, +) error + +// Start a custom controller with given parameters +func Start( + ctx context.Context, + schemes runtime.SchemeBuilder, + opts *Options, + startFunc StartFunc, +) error { + log := setupLogging(opts.VerboseMode, opts.DevLogger) + ctx = logging.NewContext(ctx, log) + + cfg, err := buildRestConfig(opts) + if err != nil { + return fmt.Errorf("building k8s client config: %w", err) + } + + mgr, err := buildManager(cfg, log, schemes, opts) + if err != nil { + return fmt.Errorf("building manager: %w", err) + } + + if err := startFunc(ctx, mgr); err != nil { + return fmt.Errorf("running start func: %w", err) + } + + log.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + return fmt.Errorf("starting manager: %w", err) + } + + return nil +} + +func buildManager(cfg *rest.Config, log *zap.SugaredLogger, schemes runtime.SchemeBuilder, opts *Options) (manager.Manager, error) { + mgr, err := manager.New( + cfg, + manager.Options{ + HealthProbeBindAddress: opts.HealthAddr, + Metrics: server.Options{BindAddress: opts.MetricsAddr}, + Logger: zapr.NewLogger(log.Desugar()), + Cache: cache.Options{ + SyncPeriod: &opts.SyncPeriod, + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("constructing manager: %w", err) + } + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + return nil, fmt.Errorf("adding healthz: %w", err) + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + return nil, fmt.Errorf("adding readyz: %w", err) + } + + if schemes != nil { + if err := schemes.AddToScheme(mgr.GetScheme()); err != nil { + return nil, err + } + } + return mgr, nil +} + +func buildRestConfig(o *Options) (*rest.Config, error) { + if o.InCluster { + if o.KubeContext != "" { + return nil, errors.New(errKubeContextSetInCluster) + } + + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("building in-cluster kubeconfig: %w", err) + } + + cfg.QPS = o.ClientQPS + cfg.Burst = o.ClientBurst + + return cfg, err + } + + if o.KubeContext == "" { + return nil, errors.New(errNoValidKubeContext) + } + + var rules *clientcmd.ClientConfigLoadingRules + if o.KubeConfig != "" { + rules = &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.KubeConfig} + } else { + rules = clientcmd.NewDefaultClientConfigLoadingRules() + } + + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + rules, + &clientcmd.ConfigOverrides{ + CurrentContext: o.KubeContext, + }, + ).ClientConfig() + + cfg.QPS = o.ClientQPS + cfg.Burst = o.ClientBurst + + return cfg, err +} + +func setupLogging(verboseMode, devLogger bool) *zap.SugaredLogger { + var baseLogger *zap.Logger + if devLogger { + l, err := zap.NewDevelopment() + if err != nil { + // TODO(eac): fixme + panic(err) + } + baseLogger = l + } else { + level := zapcore.InfoLevel + if verboseMode { + level = zapcore.DebugLevel + } + atomicLevel := zap.NewAtomicLevelAt(level) + zapOpts := []zaputil.Opts{ + zaputil.Level(&atomicLevel), + func(options *zaputil.Options) { + options.TimeEncoder = zapcore.ISO8601TimeEncoder + }, + } + if devLogger { + zapOpts = append( + zapOpts, + // Only set debug mode if specified. This will use a non-json (human-readable) encoder which makes it impossible + // to use any json parsing tools for the log. Should only be enabled explicitly + zaputil.UseDevMode(true), + ) + } + baseLogger = zaputil.NewRaw(zapOpts...) + } + + // set controller-runtime global logger + ctrl.SetLogger(zapr.NewLogger(baseLogger)) + + return baseLogger.Sugar() +} diff --git a/pkg/bootstrap/bootstrap_test.go b/pkg/bootstrap/bootstrap_test.go new file mode 100644 index 0000000..ead15ee --- /dev/null +++ b/pkg/bootstrap/bootstrap_test.go @@ -0,0 +1,27 @@ +package bootstrap + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = DescribeTable("buildRestConfig should fail", + func(inCluster bool, kubeContext string, want string) { + opts := &Options{ + InCluster: inCluster, + KubeContext: kubeContext, + } + _, err := buildRestConfig(opts) + Expect(err).Should(MatchError(want)) + }, + Entry("implicitly", false, "", errNoValidKubeContext), + Entry("when both inCluster and context are set", + true, "foo", errKubeContextSetInCluster), +) + +func TestBootstrap(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Bootstrap") +} diff --git a/pkg/encoding/json/decode.go b/pkg/encoding/json/decode.go new file mode 100644 index 0000000..24c1959 --- /dev/null +++ b/pkg/encoding/json/decode.go @@ -0,0 +1,1309 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "encoding" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a value implementing the Unmarshaler interface, +// Unmarshal calls that value's UnmarshalJSON method, including +// when the input is a JSON null. +// Otherwise, if the value implements encoding.TextUnmarshaler +// and the input is a JSON quoted string, Unmarshal calls that value's +// UnmarshalText method with the unquoted form of the string. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. By +// default, object keys which don't have a corresponding struct field are +// ignored (see Decoder.DisallowUnknownFields for an alternative). +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. +// +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be any string type, an integer, implement json.Unmarshaler, or +// implement encoding.TextUnmarshaler. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshaling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// “not present,” unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +func Unmarshal(data []byte, v interface{}) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by types +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of Unmarshal itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // the full path from root node to the field +} + +func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// +// Deprecated: No longer used; kept for compatibility. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + d.scanWhile(scanSkipSpace) + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + err := d.value(rv) + if err != nil { + return d.addErrorContext(err) + } + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// An errorContext provides context for type errors during decoding. +type errorContext struct { + Struct reflect.Type + FieldStack []string +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // next read offset in data + opcode int // last read result + scan scanner + errorContext *errorContext + savedError error + useNumber bool + disallowUnknownFields bool +} + +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + +// phasePanicMsg is used as a panic message when we end up with something that +// shouldn't happen. It can indicate a bug in the JSON decoder, or that +// something is editing the data slice while the decoder executes. +const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?" + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + if d.errorContext != nil { + d.errorContext.Struct = nil + // Reuse the allocated space for the FieldStack slice. + d.errorContext.FieldStack = d.errorContext.FieldStack[:0] + } + return d +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = d.addErrorContext(err) + } +} + +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext != nil && (d.errorContext.Struct != nil || len(d.errorContext.FieldStack) > 0) { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct.Name() + err.Field = strings.Join(d.errorContext.FieldStack, ".") + } + } + return err +} + +// skip scans to the end of what was started. +func (d *decodeState) skip() { + s, data, i := &d.scan, d.data, d.off + depth := len(s.parseState) + for { + op := s.step(s, data[i]) + i++ + if len(s.parseState) < depth { + d.off = i + d.opcode = op + return + } + } +} + +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + if d.off < len(d.data) { + d.opcode = d.scan.step(&d.scan, d.data[d.off]) + d.off++ + } else { + d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(data) { + newOp := s.step(s, data[i]) + i++ + if newOp != op { + d.opcode = newOp + d.off = i + return + } + } + + d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} + +// rescanLiteral is similar to scanWhile(scanContinue), but it specialises the +// common case where we're decoding a literal. The decoder scans the input +// twice, once for syntax errors and to check the length of the value, and the +// second to perform the decoding. +// +// Only in the second step do we use decodeState to tokenize literals, so we +// know there aren't any syntax errors. We can take advantage of that knowledge, +// and scan a literal's bytes much more quickly. +func (d *decodeState) rescanLiteral() { + data, i := d.data, d.off +Switch: + switch data[i-1] { + case '"': // string + for ; i < len(data); i++ { + switch data[i] { + case '\\': + i++ // escaped char + case '"': + i++ // tokenize the closing quote too + break Switch + } + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number + for ; i < len(data); i++ { + switch data[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '.', 'e', 'E', '+', '-': + default: + break Switch + } + } + case 't': // true + i += len("rue") + case 'f': // false + i += len("alse") + case 'n': // null + i += len("ull") + } + if i < len(data) { + d.opcode = stateEndValue(&d.scan, data[i]) + } else { + d.opcode = scanEnd + } + d.off = i + 1 +} + +// value consumes a JSON value from d.data[d.off-1:], decoding into v, and +// reads the following byte ahead. If v is invalid, the value is discarded. +// The first byte of the value has been read already. +func (d *decodeState) value(v reflect.Value) error { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray: + if v.IsValid() { + if err := d.array(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginObject: + if v.IsValid() { + if err := d.object(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginLiteral: + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + if v.IsValid() { + if err := d.literalStore(d.data[start:d.readIndex()], v, false); err != nil { + return err + } + } + } + return nil +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() interface{} { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray, scanBeginObject: + d.skip() + d.scanNext() + + case scanBeginLiteral: + v := d.literalInterface() + switch v.(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// If it encounters an Unmarshaler, indirect stops and returns that. +// If decodingNull is true, indirect stops at the first settable pointer so it +// can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // Issue #24153 indicates that it is generally not a guaranteed property + // that you may round-trip a reflect.Value by calling Value.Addr().Elem() + // and expect the value to still be settable for values derived from + // unexported embedded struct fields. + // + // The logic below effectively does this when it first addresses the value + // (to satisfy possible pointer methods) and continues to dereference + // subsequent pointers as necessary. + // + // After the first round-trip, we set v back to the original value to + // preserve the original RW flags contained in reflect.Value. + v0 := v + haveAddr := false + + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + haveAddr = true + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + haveAddr = false + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if decodingNull && v.CanSet() { + break + } + + // Prevent infinite loop if v is an interface pointing to its own address: + // var v interface{} + // v = &v + if v.Elem().Kind() == reflect.Interface && v.Elem().Elem() == v { + v = v.Elem() + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 && v.CanInterface() { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + } + + if haveAddr { + v = v0 // restore original value after round-trip Value.Addr().Elem() + haveAddr = false + } else { + v = v.Elem() + } + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into v. +// The first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + ai := d.arrayInterface() + v.Set(reflect.ValueOf(ai)) + return nil + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + case reflect.Array, reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + // Get element of array, growing if necessary. + if v.Kind() == reflect.Slice { + // Grow slice if necessary + if i >= v.Cap() { + newcap := v.Cap() + v.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) + reflect.Copy(newv, v) + v.Set(newv) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + if err := d.value(v.Index(i)); err != nil { + return err + } + } else { + // Ran out of fixed array: skip. + if err := d.value(reflect.Value{}); err != nil { + return err + } + } + i++ + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + // Array. Zero the rest. + z := reflect.Zero(v.Type().Elem()) + for ; i < v.Len(); i++ { + v.Index(i).Set(z) + } + } else { + v.SetLen(i) + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + return nil +} + +var nullLiteral = []byte("null") +var textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() + +// object consumes an object from d.data[d.off-1:], decoding into v. +// The first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + t := v.Type() + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + oi := d.objectInterface() + v.Set(reflect.ValueOf(oi)) + return nil + } + + var fields structFields + + // Check type of target: + // struct or + // map[T1]T2 where T1 is string, an integer type, + // or an encoding.TextUnmarshaler + switch v.Kind() { + case reflect.Map: + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + fields = cachedTypeFields(t) + // ok + default: + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + + var mapElem reflect.Value + var origErrorContext errorContext + if d.errorContext != nil { + origErrorContext = *d.errorContext + } + + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquoteBytes(item) + if !ok { + panic(phasePanicMsg) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := t.Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.Set(reflect.Zero(elemType)) + } + subv = mapElem + } else { + var f *field + if i, ok := fields.nameIndex[string(key)]; ok { + // Found an exact name match. + f = &fields.list[i] + } else { + // Fall back to the expensive case-insensitive + // linear search. + for i := range fields.list { + ff := &fields.list[i] + if ff.equalFold(ff.nameBytes, key) { + f = ff + break + } + } + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + // If a struct embeds a pointer to an unexported type, + // it is not possible to set a newly allocated value + // since the field is unexported. + // + // See https://golang.org/issue/21357 + if !subv.CanSet() { + d.saveError(fmt.Errorf("json: cannot set embedded pointer to unexported struct: %v", subv.Type().Elem())) + // Invalidate subv to ensure d.value(subv) skips over + // the JSON value without assigning it to subv. + subv = reflect.Value{} + destring = false + break + } + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + if d.errorContext == nil { + d.errorContext = new(errorContext) + } + d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) + d.errorContext.Struct = t + } else if d.disallowUnknownFields { + d.saveError(fmt.Errorf("json: unknown field %q", key)) + } + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + if err := d.literalStore(nullLiteral, subv, false); err != nil { + return err + } + case string: + if err := d.literalStore([]byte(qv), subv, true); err != nil { + return err + } + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) + } + } else { + if err := d.value(subv); err != nil { + return err + } + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kt := t.Key() + var kv reflect.Value + switch { + case reflect.PtrTo(kt).Implements(textUnmarshalerType): + kv = reflect.New(kt) + if err := d.literalStore(item, kv, true); err != nil { + return err + } + kv = kv.Elem() + case kt.Kind() == reflect.String: + kv = reflect.ValueOf(key).Convert(kt) + default: + switch kt.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := string(key) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + s := string(key) + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + default: + panic("json: Unexpected key type") // should never occur + } + } + if kv.IsValid() { + v.SetMapIndex(kv, subv) + } + } + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.errorContext != nil { + // Reset errorContext to its original state. + // Keep the same underlying array for FieldStack, to reuse the + // space and avoid unnecessary allocs. + d.errorContext.FieldStack = d.errorContext.FieldStack[:len(origErrorContext.FieldStack)] + d.errorContext.Struct = origErrorContext.Struct + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return nil +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeOf(0.0), Offset: int64(d.off)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) error { + // Check for unmarshaler. + if len(item) == 0 { + //Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + isNull := item[0] == 'n' // null + u, ut, pv := indirect(v, isNull) + if u != nil { + return u.UnmarshalJSON(item) + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + val := "number" + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) + return nil + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + return ut.UnmarshalText(s) + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + v.Set(reflect.Zero(v.Type())) + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.SetBytes(b[:n]) + case reflect.String: + if v.Type() == numberType && !isValidNumber(string(s)) { + return fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item) + } + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + s := string(item) + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + // s must be a valid number, because it's + // already been tokenized. + v.SetString(s) + break + } + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetFloat(n) + } + } + return nil +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() (val interface{}) { + switch d.opcode { + default: + panic(phasePanicMsg) + case scanBeginArray: + val = d.arrayInterface() + d.scanNext() + case scanBeginObject: + val = d.objectInterface() + d.scanNext() + case scanBeginLiteral: + val = d.literalInterface() + } + return +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read string key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return m +} + +// literalInterface consumes and returns a literal from d.data[d.off-1:] and +// it reads the following byte ahead. The first byte of the literal has been +// read already (that's how the caller knows it's a literal). +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + item := d.data[start:d.readIndex()] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + panic(phasePanicMsg) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var r rune + for _, c := range s[2:6] { + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = c - 'a' + 10 + case 'A' <= c && c <= 'F': + c = c - 'A' + 10 + default: + return -1 + } + r = r*16 + rune(c) + } + return r +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/pkg/encoding/json/decode_test.go b/pkg/encoding/json/decode_test.go new file mode 100644 index 0000000..219e845 --- /dev/null +++ b/pkg/encoding/json/decode_test.go @@ -0,0 +1,2574 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "encoding" + "errors" + "fmt" + "image" + "math" + "math/big" + "net" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +type T struct { + X string + Y int + Z int `json:"-"` +} + +type U struct { + Alphabet string `json:"alpha"` +} + +type V struct { + F1 interface{} + F2 int32 + F3 Number + F4 *VOuter +} + +type VOuter struct { + V V +} + +type W struct { + S SS +} + +type P struct { + PP PP +} + +type PP struct { + T T + Ts []T +} + +type SS string + +func (*SS) UnmarshalJSON(data []byte) error { + return &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS(""))} +} + +// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and +// without UseNumber +var ifaceNumAsFloat64 = map[string]interface{}{ + "k1": float64(1), + "k2": "s", + "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, +} + +var ifaceNumAsNumber = map[string]interface{}{ + "k1": Number("1"), + "k2": "s", + "k3": []interface{}{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, +} + +type tx struct { + x int +} + +type u8 uint8 + +// A type that can unmarshal itself. + +type unmarshaler struct { + T bool +} + +func (u *unmarshaler) UnmarshalJSON(b []byte) error { + *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. + return nil +} + +type ustruct struct { + M unmarshaler +} + +type unmarshalerText struct { + A, B string +} + +// needed for re-marshaling tests +func (u unmarshalerText) MarshalText() ([]byte, error) { + return []byte(u.A + ":" + u.B), nil +} + +func (u *unmarshalerText) UnmarshalText(b []byte) error { + pos := bytes.IndexByte(b, ':') + if pos == -1 { + return errors.New("missing separator") + } + u.A, u.B = string(b[:pos]), string(b[pos+1:]) + return nil +} + +var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) + +type ustructText struct { + M unmarshalerText +} + +// u8marshal is an integer type that can marshal/unmarshal itself. +type u8marshal uint8 + +func (u8 u8marshal) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("u%d", u8)), nil +} + +var errMissingU8Prefix = errors.New("missing 'u' prefix") + +func (u8 *u8marshal) UnmarshalText(b []byte) error { + if !bytes.HasPrefix(b, []byte{'u'}) { + return errMissingU8Prefix + } + n, err := strconv.Atoi(string(b[1:])) + if err != nil { + return err + } + *u8 = u8marshal(n) + return nil +} + +var _ encoding.TextUnmarshaler = (*u8marshal)(nil) + +var ( + umtrue = unmarshaler{true} + umslice = []unmarshaler{{true}} + umstruct = ustruct{unmarshaler{true}} + + umtrueXY = unmarshalerText{"x", "y"} + umsliceXY = []unmarshalerText{{"x", "y"}} + umstructXY = ustructText{unmarshalerText{"x", "y"}} + + ummapXY = map[unmarshalerText]bool{{"x", "y"}: true} +) + +// Test data structures for anonymous fields. + +type Point struct { + Z int +} + +type Top struct { + Level0 int + Embed0 + *Embed0a + *Embed0b `json:"e,omitempty"` // treated as named + Embed0c `json:"-"` // ignored + Loop + Embed0p // has Point with X, Y, used + Embed0q // has Point with Z, used + embed // contains exported field +} + +type Embed0 struct { + Level1a int // overridden by Embed0a's Level1a with json tag + Level1b int // used because Embed0a's Level1b is renamed + Level1c int // used because Embed0a's Level1c is ignored + Level1d int // annihilated by Embed0a's Level1d + Level1e int `json:"x"` // annihilated by Embed0a.Level1e +} + +type Embed0a struct { + Level1a int `json:"Level1a,omitempty"` + Level1b int `json:"LEVEL1B,omitempty"` + Level1c int `json:"-"` + Level1d int // annihilated by Embed0's Level1d + Level1f int `json:"x"` // annihilated by Embed0's Level1e +} + +type Embed0b Embed0 + +type Embed0c Embed0 + +type Embed0p struct { + image.Point +} + +type Embed0q struct { + Point +} + +type embed struct { + Q int +} + +type Loop struct { + Loop1 int `json:",omitempty"` + Loop2 int `json:",omitempty"` + *Loop +} + +// From reflect test: +// The X in S6 and S7 annihilate, but they also block the X in S8.S9. +type S5 struct { + S6 + S7 + S8 +} + +type S6 struct { + X int +} + +type S7 S6 + +type S8 struct { + S9 +} + +type S9 struct { + X int + Y int +} + +// From reflect test: +// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. +type S10 struct { + S11 + S12 + S13 +} + +type S11 struct { + S6 +} + +type S12 struct { + S6 +} + +type S13 struct { + S8 +} + +type Ambig struct { + // Given "hello", the first match should win. + First int `json:"HELLO"` + Second int `json:"Hello"` +} + +type XYZ struct { + X interface{} + Y interface{} + Z interface{} +} + +type unexportedWithMethods struct{} + +func (unexportedWithMethods) F() {} + +type byteWithMarshalJSON byte + +func (b byteWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, byte(b))), nil +} + +func (b *byteWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalJSON(i) + return nil +} + +type byteWithPtrMarshalJSON byte + +func (b *byteWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return byteWithMarshalJSON(*b).MarshalJSON() +} + +func (b *byteWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*byteWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type byteWithMarshalText byte + +func (b byteWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, byte(b))), nil +} + +func (b *byteWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalText(i) + return nil +} + +type byteWithPtrMarshalText byte + +func (b *byteWithPtrMarshalText) MarshalText() ([]byte, error) { + return byteWithMarshalText(*b).MarshalText() +} + +func (b *byteWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*byteWithMarshalText)(b).UnmarshalText(data) +} + +type intWithMarshalJSON int + +func (b intWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, int(b))), nil +} + +func (b *intWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalJSON(i) + return nil +} + +type intWithPtrMarshalJSON int + +func (b *intWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return intWithMarshalJSON(*b).MarshalJSON() +} + +func (b *intWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*intWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type intWithMarshalText int + +func (b intWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, int(b))), nil +} + +func (b *intWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalText(i) + return nil +} + +type intWithPtrMarshalText int + +func (b *intWithPtrMarshalText) MarshalText() ([]byte, error) { + return intWithMarshalText(*b).MarshalText() +} + +func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*intWithMarshalText)(b).UnmarshalText(data) +} + +type mapStringToStringData struct { + Data map[string]string `json:"data"` +} + +type unmarshalTest struct { + in string + ptr interface{} // new(type) + out interface{} + err error + useNumber bool + golden bool + disallowUnknownFields bool +} + +type B struct { + B bool `json:",string"` +} + +type DoublePtr struct { + I **int + J **int +} + +var unmarshalTests = []unmarshalTest{ + // basic types + {in: `true`, ptr: new(bool), out: true}, + {in: `1`, ptr: new(int), out: 1}, + {in: `1.2`, ptr: new(float64), out: 1.2}, + {in: `-5`, ptr: new(int16), out: int16(-5)}, + {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, + {in: `2`, ptr: new(Number), out: Number("2")}, + {in: `2`, ptr: new(interface{}), out: float64(2.0)}, + {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, + {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {in: "null", ptr: new(interface{}), out: nil}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7, "T", "X"}}, + {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(""), 8, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(SS("")), 0, "W", "S"}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, + + // raw values with whitespace + {in: "\n true ", ptr: new(bool), out: true}, + {in: "\t 1 ", ptr: new(int), out: 1}, + {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // Z has a "-" tag. + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), err: fmt.Errorf("json: unknown field \"Z\""), disallowUnknownFields: true}, + + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true}, + + // syntax errors + {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, + {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, + {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, + {in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}}, + {in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}}, + + // raw value errors + {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}}, + {in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}}, + {in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}}, + {in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}}, + + // array tests + {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, + + // empty array to interface test + {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, + {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, + {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, + {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, + + // composite tests + {in: allValueIndent, ptr: new(All), out: allValue}, + {in: allValueCompact, ptr: new(All), out: allValue}, + {in: allValueIndent, ptr: new(*All), out: &allValue}, + {in: allValueCompact, ptr: new(*All), out: &allValue}, + {in: pallValueIndent, ptr: new(All), out: pallValue}, + {in: pallValueCompact, ptr: new(All), out: pallValue}, + {in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // unmarshal interface test + {in: `{"T":false}`, ptr: new(unmarshaler), out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {in: `{"T":false}`, ptr: new(*unmarshaler), out: &umtrue}, + {in: `[{"T":false}]`, ptr: new([]unmarshaler), out: umslice}, + {in: `[{"T":false}]`, ptr: new(*[]unmarshaler), out: &umslice}, + {in: `{"M":{"T":"x:y"}}`, ptr: new(ustruct), out: umstruct}, + + // UnmarshalText interface test + {in: `"x:y"`, ptr: new(unmarshalerText), out: umtrueXY}, + {in: `"x:y"`, ptr: new(*unmarshalerText), out: &umtrueXY}, + {in: `["x:y"]`, ptr: new([]unmarshalerText), out: umsliceXY}, + {in: `["x:y"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY}, + {in: `{"M":"x:y"}`, ptr: new(ustructText), out: umstructXY}, + + // integer-keyed map test + { + in: `{"-1":"a","0":"b","1":"c"}`, + ptr: new(map[int]string), + out: map[int]string{-1: "a", 0: "b", 1: "c"}, + }, + { + in: `{"0":"a","10":"c","9":"b"}`, + ptr: new(map[u8]string), + out: map[u8]string{0: "a", 9: "b", 10: "c"}, + }, + { + in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + ptr: new(map[int64]string), + out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + }, + { + in: `{"18446744073709551615":"max"}`, + ptr: new(map[uint64]string), + out: map[uint64]string{math.MaxUint64: "max"}, + }, + { + in: `{"0":false,"10":true}`, + ptr: new(map[uintptr]bool), + out: map[uintptr]bool{0: false, 10: true}, + }, + + // Check that MarshalText and UnmarshalText take precedence + // over default integer handling in map keys. + { + in: `{"u2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{2: 4}, + }, + { + in: `{"2":4}`, + ptr: new(map[u8marshal]int), + err: errMissingU8Prefix, + }, + + // integer-keyed map errors + { + in: `{"abc":"abc"}`, + ptr: new(map[int]string), + err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, + }, + { + in: `{"256":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + { + in: `{"128":"abc"}`, + ptr: new(map[int8]string), + err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, + }, + { + in: `{"-1":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[int]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(int(0)), Offset: 7}, + }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[uint]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, + }, + + // Map keys can be encoding.TextUnmarshalers. + {in: `{"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + // If multiple values for the same key exists, only the most recent value is used. + {in: `{"x:y":false,"x:y":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY}, + + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + }, + }, + { + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + err: fmt.Errorf("json: unknown field \"X\""), + disallowUnknownFields: true, + }, + { + in: `{"I": 0, "I": null, "J": null}`, + ptr: new(DoublePtr), + out: DoublePtr{I: nil, J: nil}, + }, + + // invalid UTF-8 is coerced to valid UTF-8. + { + in: "\"hello\xffworld\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + in: "\"hello\xc2\xc2world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\xc2\xffworld\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\\ud800world\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + ptr: new(string), + out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", + }, + + // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[time.Time]string), + out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + }, + + // issue 8305 + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: new(map[Point]string), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, + }, + { + in: `{"asdf": "hello world"}`, + ptr: new(map[unmarshaler]string), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, + }, + + // related to issue 13783. + // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, + // similar to marshaling a slice of typed int. + // These tests check that, assuming the byte type also has valid decoding methods, + // either the old base64 string encoding or the new per-element encoding can be + // successfully unmarshaled. The custom unmarshalers were accessible in earlier + // versions of Go, even though the custom marshaler was not. + { + in: `"AQID"`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + // ints work with the marshaler but not the base64 []byte case + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalJSON), + out: []intWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalText), + out: []intWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalJSON), + out: []intWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalText), + out: []intWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeOf(int32(0)), + Offset: 20, + }, + }, + { + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "V.F2", + Type: reflect.TypeOf(int32(0)), + Offset: 30, + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, + {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, + + // additional tests for disallowUnknownFields + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18, + "extra": true + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12, + "extra": null + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17, + "Q": 18 + }`, + ptr: new(Top), + err: fmt.Errorf("json: unknown field \"extra\""), + disallowUnknownFields: true, + }, + // issue 26444 + // UnmarshalTypeError without field & struct values + { + in: `{"data":{"test1": "bob", "test2": 123}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 37, Struct: "mapStringToStringData", Field: "data"}, + }, + { + in: `{"data":{"test1": 123, "test2": "bob"}}`, + ptr: new(mapStringToStringData), + err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, + }, + + // trying to decode JSON arrays or objects via TextUnmarshaler + { + in: `[1, 2, 3]`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, + { + in: `{"foo": "bar"}`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, + // #22369 + { + in: `{"PP": {"T": {"Y": "bad-type"}}}`, + ptr: new(P), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "PP.T.Y", + Type: reflect.TypeOf(int(0)), + Offset: 29, + }, + }, + { + in: `{"Ts": [{"Y": 1}, {"Y": 2}, {"Y": "bad-type"}]}`, + ptr: new(PP), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "T", + Field: "Ts.Y", + Type: reflect.TypeOf(int(0)), + Offset: 29, + }, + }, + // #14702 + { + in: `invalid`, + ptr: new(Number), + err: &SyntaxError{ + msg: "invalid character 'i' looking for beginning of value", + Offset: 1, + }, + }, + { + in: `"invalid"`, + ptr: new(Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(struct{ A Number }), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(struct { + A Number `json:",string"` + }), + err: fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number", `invalid`), + }, + { + in: `{"A":"invalid"}`, + ptr: new(map[string]Number), + err: fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", `"invalid"`), + }, +} + +func TestMarshal(t *testing.T) { + b, err := Marshal(allValue) + if err != nil { + t.Fatalf("Marshal allValue: %v", err) + } + if string(b) != allValueCompact { + t.Errorf("Marshal allValueCompact") + diff(t, b, []byte(allValueCompact)) + return + } + + b, err = Marshal(pallValue) + if err != nil { + t.Fatalf("Marshal pallValue: %v", err) + } + if string(b) != pallValueCompact { + t.Errorf("Marshal pallValueCompact") + diff(t, b, []byte(pallValueCompact)) + return + } +} + +var badUTF8 = []struct { + in, out string +}{ + {"hello\xffworld", `"hello\ufffdworld"`}, + {"", `""`}, + {"\xff", `"\ufffd"`}, + {"\xff\xff", `"\ufffd\ufffd"`}, + {"a\xffb", `"a\ufffdb"`}, + {"\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"日本\ufffd\ufffd\ufffd"`}, +} + +func TestMarshalBadUTF8(t *testing.T) { + for _, tt := range badUTF8 { + b, err := Marshal(tt.in) + if string(b) != tt.out || err != nil { + t.Errorf("Marshal(%q) = %#q, %v, want %#q, nil", tt.in, b, err, tt.out) + } + } +} + +func TestMarshalNumberZeroVal(t *testing.T) { + var n Number + out, err := Marshal(n) + if err != nil { + t.Fatal(err) + } + outStr := string(out) + if outStr != "0" { + t.Fatalf("Invalid zero val for Number: %q", outStr) + } +} + +func TestMarshalEmbeds(t *testing.T) { + top := &Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + embed: embed{ + Q: 18, + }, + } + b, err := Marshal(top) + if err != nil { + t.Fatal(err) + } + want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17,\"Q\":18}" + if string(b) != want { + t.Errorf("Wrong marshal result.\n got: %q\nwant: %q", b, want) + } +} + +func equalError(a, b error) bool { + if a == nil { + return b == nil + } + if b == nil { + return a == nil + } + return a.Error() == b.Error() +} + +func TestUnmarshal(t *testing.T) { + for i, tt := range unmarshalTests { + var scan scanner + in := []byte(tt.in) + if err := checkValid(in, &scan); err != nil { + if !equalError(err, tt.err) { + t.Errorf("#%d: checkValid: %#v", i, err) + continue + } + } + if tt.ptr == nil { + continue + } + + typ := reflect.TypeOf(tt.ptr) + if typ.Kind() != reflect.Ptr { + t.Errorf("#%d: unmarshalTest.ptr %T is not a pointer type", i, tt.ptr) + continue + } + typ = typ.Elem() + + // v = new(right-type) + v := reflect.New(typ) + + if !reflect.DeepEqual(tt.ptr, v.Interface()) { + // There's no reason for ptr to point to non-zero data, + // as we decode into new(right-type), so the data is + // discarded. + // This can easily mean tests that silently don't test + // what they should. To test decoding into existing + // data, see TestPrefilled. + t.Errorf("#%d: unmarshalTest.ptr %#v is not a pointer to a zero value", i, tt.ptr) + continue + } + + dec := NewDecoder(bytes.NewReader(in)) + if tt.useNumber { + dec.UseNumber() + } + if tt.disallowUnknownFields { + dec.DisallowUnknownFields() + } + if err := dec.Decode(v.Interface()); !equalError(err, tt.err) { + t.Errorf("#%d: %v, want %v", i, err, tt.err) + continue + } else if err != nil { + continue + } + if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), tt.out) + data, _ := Marshal(v.Elem().Interface()) + println(string(data)) + data, _ = Marshal(tt.out) + println(string(data)) + continue + } + + // Check round trip also decodes correctly. + if tt.err == nil { + enc, err := Marshal(v.Interface()) + if err != nil { + t.Errorf("#%d: error re-marshaling: %v", i, err) + continue + } + if tt.golden && !bytes.Equal(enc, in) { + t.Errorf("#%d: remarshal mismatch:\nhave: %s\nwant: %s", i, enc, in) + } + vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = NewDecoder(bytes.NewReader(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv.Interface()); err != nil { + t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err) + continue + } + if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), vv.Elem().Interface()) + t.Errorf(" In: %q", strings.Map(noSpace, string(in))) + t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) + continue + } + } + } +} + +func TestUnmarshalMarshal(t *testing.T) { + initBig() + var v interface{} + if err := Unmarshal(jsonBig, &v); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + b, err := Marshal(v) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if !bytes.Equal(jsonBig, b) { + t.Errorf("Marshal jsonBig") + diff(t, b, jsonBig) + return + } +} + +var numberTests = []struct { + in string + i int64 + intErr string + f float64 + floatErr string +}{ + {in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, + {in: "-12", i: -12, f: -12.0}, + {in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, +} + +// Independent of Decode, basic coverage of the accessors in Number +func TestNumberAccessors(t *testing.T) { + for _, tt := range numberTests { + n := Number(tt.in) + if s := n.String(); s != tt.in { + t.Errorf("Number(%q).String() is %q", tt.in, s) + } + if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { + t.Errorf("Number(%q).Int64() is %d", tt.in, i) + } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { + t.Errorf("Number(%q).Int64() wanted error %q but got: %v", tt.in, tt.intErr, err) + } + if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { + t.Errorf("Number(%q).Float64() is %g", tt.in, f) + } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { + t.Errorf("Number(%q).Float64() wanted error %q but got: %v", tt.in, tt.floatErr, err) + } + } +} + +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !bytes.Equal(s0, s1) { + t.Errorf("Marshal large byte slice") + diff(t, s0, s1) + } +} + +type Xint struct { + X int +} + +func TestUnmarshalInterface(t *testing.T) { + var xint Xint + var i interface{} = &xint + if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("Did not write to xint") + } +} + +func TestUnmarshalPtrPtr(t *testing.T) { + var xint Xint + pxint := &xint + if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("Did not write to xint") + } +} + +func TestEscape(t *testing.T) { + const input = `"foobar"` + " [\u2028 \u2029]" + const expected = `"\"foobar\"\u003chtml\u003e [\u2028 \u2029]"` + b, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if s := string(b); s != expected { + t.Errorf("Encoding of [%s]:\n got [%s]\nwant [%s]", input, s, expected) + } +} + +// WrongString is a struct that's misusing the ,string modifier. +type WrongString struct { + Message string `json:"result,string"` +} + +type wrongStringTest struct { + in, err string +} + +var wrongStringTests = []wrongStringTest{ + {`{"result":"x"}`, `json: invalid use of ,string struct tag, trying to unmarshal "x" into string`}, + {`{"result":"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "foo" into string`}, + {`{"result":"123"}`, `json: invalid use of ,string struct tag, trying to unmarshal "123" into string`}, + {`{"result":123}`, `json: invalid use of ,string struct tag, trying to unmarshal unquoted value into string`}, + {`{"result":"\""}`, `json: invalid use of ,string struct tag, trying to unmarshal "\"" into string`}, + {`{"result":"\"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "\"foo" into string`}, +} + +// If people misuse the ,string modifier, the error message should be +// helpful, telling the user that they're doing it wrong. +func TestErrorMessageFromMisusedString(t *testing.T) { + for n, tt := range wrongStringTests { + r := strings.NewReader(tt.in) + var s WrongString + err := NewDecoder(r).Decode(&s) + got := fmt.Sprintf("%v", err) + if got != tt.err { + t.Errorf("%d. got err = %q, want %q", n, got, tt.err) + } + } +} + +func noSpace(c rune) rune { + if isSpace(byte(c)) { //only used for ascii + return -1 + } + return c +} + +type All struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + + Foo string `json:"bar"` + Foo2 string `json:"bar2,dummyopt"` + + IntStr int64 `json:",string"` + UintptrStr uintptr `json:",string"` + + PBool *bool + PInt *int + PInt8 *int8 + PInt16 *int16 + PInt32 *int32 + PInt64 *int64 + PUint *uint + PUint8 *uint8 + PUint16 *uint16 + PUint32 *uint32 + PUint64 *uint64 + PUintptr *uintptr + PFloat32 *float32 + PFloat64 *float64 + + String string + PString *string + + Map map[string]Small + MapP map[string]*Small + PMap *map[string]Small + PMapP *map[string]*Small + + EmptyMap map[string]Small + NilMap map[string]Small + + Slice []Small + SliceP []*Small + PSlice *[]Small + PSliceP *[]*Small + + EmptySlice []Small + NilSlice []Small + + StringSlice []string + ByteSlice []byte + + Small Small + PSmall *Small + PPSmall **Small + + Interface interface{} + PInterface *interface{} + + unexported int +} + +type Small struct { + Tag string +} + +var allValue = All{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Uintptr: 12, + Float32: 14.1, + Float64: 15.1, + Foo: "foo", + Foo2: "foo2", + IntStr: 42, + UintptrStr: 44, + String: "16", + Map: map[string]Small{ + "17": {Tag: "tag17"}, + "18": {Tag: "tag18"}, + }, + MapP: map[string]*Small{ + "19": {Tag: "tag19"}, + "20": nil, + }, + EmptyMap: map[string]Small{}, + Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, + SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, + EmptySlice: []Small{}, + StringSlice: []string{"str24", "str25", "str26"}, + ByteSlice: []byte{27, 28, 29}, + Small: Small{Tag: "tag30"}, + PSmall: &Small{Tag: "tag31"}, + Interface: 5.2, +} + +var pallValue = All{ + PBool: &allValue.Bool, + PInt: &allValue.Int, + PInt8: &allValue.Int8, + PInt16: &allValue.Int16, + PInt32: &allValue.Int32, + PInt64: &allValue.Int64, + PUint: &allValue.Uint, + PUint8: &allValue.Uint8, + PUint16: &allValue.Uint16, + PUint32: &allValue.Uint32, + PUint64: &allValue.Uint64, + PUintptr: &allValue.Uintptr, + PFloat32: &allValue.Float32, + PFloat64: &allValue.Float64, + PString: &allValue.String, + PMap: &allValue.Map, + PMapP: &allValue.MapP, + PSlice: &allValue.Slice, + PSliceP: &allValue.SliceP, + PPSmall: &allValue.PSmall, + PInterface: &allValue.Interface, +} + +var allValueIndent = `{ + "Bool": true, + "Int": 2, + "Int8": 3, + "Int16": 4, + "Int32": 5, + "Int64": 6, + "Uint": 7, + "Uint8": 8, + "Uint16": 9, + "Uint32": 10, + "Uint64": 11, + "Uintptr": 12, + "Float32": 14.1, + "Float64": 15.1, + "bar": "foo", + "bar2": "foo2", + "IntStr": "42", + "UintptrStr": "44", + "PBool": null, + "PInt": null, + "PInt8": null, + "PInt16": null, + "PInt32": null, + "PInt64": null, + "PUint": null, + "PUint8": null, + "PUint16": null, + "PUint32": null, + "PUint64": null, + "PUintptr": null, + "PFloat32": null, + "PFloat64": null, + "String": "16", + "PString": null, + "Map": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "MapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "PMap": null, + "PMapP": null, + "EmptyMap": {}, + "NilMap": null, + "Slice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "SliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "PSlice": null, + "PSliceP": null, + "EmptySlice": [], + "NilSlice": null, + "StringSlice": [ + "str24", + "str25", + "str26" + ], + "ByteSlice": "Gxwd", + "Small": { + "Tag": "tag30" + }, + "PSmall": { + "Tag": "tag31" + }, + "PPSmall": null, + "Interface": 5.2, + "PInterface": null +}` + +var allValueCompact = strings.Map(noSpace, allValueIndent) + +var pallValueIndent = `{ + "Bool": false, + "Int": 0, + "Int8": 0, + "Int16": 0, + "Int32": 0, + "Int64": 0, + "Uint": 0, + "Uint8": 0, + "Uint16": 0, + "Uint32": 0, + "Uint64": 0, + "Uintptr": 0, + "Float32": 0, + "Float64": 0, + "bar": "", + "bar2": "", + "IntStr": "0", + "UintptrStr": "0", + "PBool": true, + "PInt": 2, + "PInt8": 3, + "PInt16": 4, + "PInt32": 5, + "PInt64": 6, + "PUint": 7, + "PUint8": 8, + "PUint16": 9, + "PUint32": 10, + "PUint64": 11, + "PUintptr": 12, + "PFloat32": 14.1, + "PFloat64": 15.1, + "String": "", + "PString": "16", + "Map": null, + "MapP": null, + "PMap": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "PMapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "EmptyMap": null, + "NilMap": null, + "Slice": null, + "SliceP": null, + "PSlice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "PSliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "EmptySlice": null, + "NilSlice": null, + "StringSlice": null, + "ByteSlice": null, + "Small": { + "Tag": "" + }, + "PSmall": null, + "PPSmall": { + "Tag": "tag31" + }, + "Interface": null, + "PInterface": 5.2 +}` + +var pallValueCompact = strings.Map(noSpace, pallValueIndent) + +func TestRefUnmarshal(t *testing.T) { + type S struct { + // Ref is defined in encode_test.go. + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + } + want := S{ + R0: 12, + R1: new(Ref), + R2: 13, + R3: new(RefText), + } + *want.R1 = 12 + *want.R3 = 13 + + var got S + if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } +} + +// Test that the empty string doesn't panic decoding when ,string is specified +// Issue 3450 +func TestEmptyString(t *testing.T) { + type T2 struct { + Number1 int `json:",string"` + Number2 int `json:",string"` + } + data := `{"Number1":"1", "Number2":""}` + dec := NewDecoder(strings.NewReader(data)) + var t2 T2 + err := dec.Decode(&t2) + if err == nil { + t.Fatal("Decode: did not return error") + } + if t2.Number1 != 1 { + t.Fatal("Decode: did not set Number1") + } +} + +// Test that a null for ,string is not replaced with the previous quoted string (issue 7046). +// It should also not be an error (issue 2540, issue 8587). +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + C *int `json:",string"` + } + data := []byte(`{"A": "1", "B": null, "C": null}`) + var s T + s.B = 1 + s.C = new(int) + *s.C = 2 + err := Unmarshal(data, &s) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if s.B != 1 || s.C != nil { + t.Fatalf("after Unmarshal, s.B=%d, s.C=%p, want 1, nil", s.B, s.C) + } +} + +func intp(x int) *int { + p := new(int) + *p = x + return p +} + +func intpp(x *int) **int { + pp := new(*int) + *pp = x + return pp +} + +var interfaceSetTests = []struct { + pre interface{} + json string + post interface{} +}{ + {"foo", `"bar"`, "bar"}, + {"foo", `2`, 2.0}, + {"foo", `true`, true}, + {"foo", `null`, nil}, + + {nil, `null`, nil}, + {new(int), `null`, nil}, + {(*int)(nil), `null`, nil}, + {new(*int), `null`, new(*int)}, + {(**int)(nil), `null`, nil}, + {intp(1), `null`, nil}, + {intpp(nil), `null`, intpp(nil)}, + {intpp(intp(1)), `null`, intpp(nil)}, +} + +func TestInterfaceSet(t *testing.T) { + for _, tt := range interfaceSetTests { + b := struct{ X interface{} }{tt.pre} + blob := `{"X":` + tt.json + `}` + if err := Unmarshal([]byte(blob), &b); err != nil { + t.Errorf("Unmarshal %#q: %v", blob, err) + continue + } + if !reflect.DeepEqual(b.X, tt.post) { + t.Errorf("Unmarshal %#q into %#v: X=%#v, want %#v", blob, tt.pre, b.X, tt.post) + } + } +} + +type NullTest struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + String string + PBool *bool + Map map[string]string + Slice []string + Interface interface{} + + PRaw *RawMessage + PTime *time.Time + PBigInt *big.Int + PText *MustNotUnmarshalText + PBuffer *bytes.Buffer // has methods, just not relevant ones + PStruct *struct{} + + Raw RawMessage + Time time.Time + BigInt big.Int + Text MustNotUnmarshalText + Buffer bytes.Buffer + Struct struct{} +} + +// JSON null values should be ignored for primitives and string values instead of resulting in an error. +// Issue 2540 +func TestUnmarshalNulls(t *testing.T) { + // Unmarshal docs: + // The JSON null value unmarshals into an interface, map, pointer, or slice + // by setting that Go value to nil. Because null is often used in JSON to mean + // ``not present,'' unmarshaling a JSON null into any other Go type has no effect + // on the value and produces no error. + + jsonData := []byte(`{ + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null, + "PBool": null, + "Map": null, + "Slice": null, + "Interface": null, + "PRaw": null, + "PTime": null, + "PBigInt": null, + "PText": null, + "PBuffer": null, + "PStruct": null, + "Raw": null, + "Time": null, + "BigInt": null, + "Text": null, + "Buffer": null, + "Struct": null + }`) + nulls := NullTest{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14", + PBool: new(bool), + Map: map[string]string{}, + Slice: []string{}, + Interface: new(MustNotUnmarshalJSON), + PRaw: new(RawMessage), + PTime: new(time.Time), + PBigInt: new(big.Int), + PText: new(MustNotUnmarshalText), + PStruct: new(struct{}), + PBuffer: new(bytes.Buffer), + Raw: RawMessage("123"), + Time: time.Unix(123456789, 0), + BigInt: *big.NewInt(123), + } + + before := nulls.Time.String() + + err := Unmarshal(jsonData, &nulls) + if err != nil { + t.Errorf("Unmarshal of null values failed: %v", err) + } + if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || + nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || + nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { + t.Errorf("Unmarshal of null values affected primitives") + } + + if nulls.PBool != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBool") + } + if nulls.Map != nil { + t.Errorf("Unmarshal of null did not clear nulls.Map") + } + if nulls.Slice != nil { + t.Errorf("Unmarshal of null did not clear nulls.Slice") + } + if nulls.Interface != nil { + t.Errorf("Unmarshal of null did not clear nulls.Interface") + } + if nulls.PRaw != nil { + t.Errorf("Unmarshal of null did not clear nulls.PRaw") + } + if nulls.PTime != nil { + t.Errorf("Unmarshal of null did not clear nulls.PTime") + } + if nulls.PBigInt != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBigInt") + } + if nulls.PText != nil { + t.Errorf("Unmarshal of null did not clear nulls.PText") + } + if nulls.PBuffer != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBuffer") + } + if nulls.PStruct != nil { + t.Errorf("Unmarshal of null did not clear nulls.PStruct") + } + + if string(nulls.Raw) != "null" { + t.Errorf("Unmarshal of RawMessage null did not record null: %v", string(nulls.Raw)) + } + if nulls.Time.String() != before { + t.Errorf("Unmarshal of time.Time null set time to %v", nulls.Time.String()) + } + if nulls.BigInt.String() != "123" { + t.Errorf("Unmarshal of big.Int null set int to %v", nulls.BigInt.String()) + } +} + +type MustNotUnmarshalJSON struct{} + +func (x MustNotUnmarshalJSON) UnmarshalJSON(data []byte) error { + return errors.New("MustNotUnmarshalJSON was used") +} + +type MustNotUnmarshalText struct{} + +func (x MustNotUnmarshalText) UnmarshalText(text []byte) error { + return errors.New("MustNotUnmarshalText was used") +} + +func TestStringKind(t *testing.T) { + type stringKind string + + var m1, m2 map[stringKind]int + m1 = map[stringKind]int{ + "foo": 42, + } + + data, err := Marshal(m1) + if err != nil { + t.Errorf("Unexpected error marshaling: %v", err) + } + + err = Unmarshal(data, &m2) + if err != nil { + t.Errorf("Unexpected error unmarshaling: %v", err) + } + + if !reflect.DeepEqual(m1, m2) { + t.Error("Items should be equal after encoding and then decoding") + } +} + +// Custom types with []byte as underlying type could not be marshaled +// and then unmarshaled. +// Issue 8962. +func TestByteKind(t *testing.T) { + type byteKind []byte + + a := byteKind("hello") + + data, err := Marshal(a) + if err != nil { + t.Error(err) + } + var b byteKind + err = Unmarshal(data, &b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, b) { + t.Errorf("expected %v == %v", a, b) + } +} + +// The fix for issue 8962 introduced a regression. +// Issue 12921. +func TestSliceOfCustomByte(t *testing.T) { + type Uint8 uint8 + + a := []Uint8("hello") + + data, err := Marshal(a) + if err != nil { + t.Fatal(err) + } + var b []Uint8 + err = Unmarshal(data, &b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, b) { + t.Fatalf("expected %v == %v", a, b) + } +} + +var decodeTypeErrorTests = []struct { + dest interface{} + src string +}{ + {new(string), `{"user": "name"}`}, // issue 4628. + {new(error), `{}`}, // issue 4222 + {new(error), `[]`}, + {new(error), `""`}, + {new(error), `123`}, + {new(error), `true`}, +} + +func TestUnmarshalTypeError(t *testing.T) { + for _, item := range decodeTypeErrorTests { + err := Unmarshal([]byte(item.src), item.dest) + if _, ok := err.(*UnmarshalTypeError); !ok { + t.Errorf("expected type error for Unmarshal(%q, type %T): got %T", + item.src, item.dest, err) + } + } +} + +var unmarshalSyntaxTests = []string{ + "tru", + "fals", + "nul", + "123e", + `"hello`, + `[1,2,3`, + `{"key":1`, + `{"key":1,`, +} + +func TestUnmarshalSyntax(t *testing.T) { + var x interface{} + for _, src := range unmarshalSyntaxTests { + err := Unmarshal([]byte(src), &x) + if _, ok := err.(*SyntaxError); !ok { + t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err) + } + } +} + +// Test handling of unexported fields that should be ignored. +// Issue 4660 +type unexportedFields struct { + Name string + m map[string]interface{} `json:"-"` + m2 map[string]interface{} `json:"abcd"` + + s []int `json:"-"` +} + +func TestUnmarshalUnexported(t *testing.T) { + input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}, "s": [2, 3]}` + want := &unexportedFields{Name: "Bob"} + + out := &unexportedFields{} + err := Unmarshal([]byte(input), out) + if err != nil { + t.Errorf("got error %v, expected nil", err) + } + if !reflect.DeepEqual(out, want) { + t.Errorf("got %q, want %q", out, want) + } +} + +// Time3339 is a time.Time which encodes to and from JSON +// as an RFC 3339 time in UTC. +type Time3339 time.Time + +func (t *Time3339) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) + } + tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) + if err != nil { + return err + } + *t = Time3339(tm) + return nil +} + +func TestUnmarshalJSONLiteralError(t *testing.T) { + var t3 Time3339 + err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3) + if err == nil { + t.Fatalf("expected error; got time %v", time.Time(t3)) + } + if !strings.Contains(err.Error(), "range") { + t.Errorf("got err = %v; want out of range error", err) + } +} + +// Test that extra object elements in an array do not result in a +// "data changing underfoot" error. +// Issue 3717 +func TestSkipArrayObjects(t *testing.T) { + json := `[{}]` + var dest [0]interface{} + + err := Unmarshal([]byte(json), &dest) + if err != nil { + t.Errorf("got error %q, want nil", err) + } +} + +// Test semantics of pre-filled data, such as struct fields, map elements, +// slices, and arrays. +// Issues 4900 and 8837, among others. +func TestPrefilled(t *testing.T) { + // Values here change, cannot reuse table across runs. + var prefillTests = []struct { + in string + ptr interface{} + out interface{} + }{ + { + in: `{"X": 1, "Y": 2}`, + ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, + out: &XYZ{X: float64(1), Y: float64(2), Z: 1.5}, + }, + { + in: `{"X": 1, "Y": 2}`, + ptr: &map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}, + out: &map[string]interface{}{"X": float64(1), "Y": float64(2), "Z": 1.5}, + }, + { + in: `[2]`, + ptr: &[]int{1}, + out: &[]int{2}, + }, + { + in: `[2, 3]`, + ptr: &[]int{1}, + out: &[]int{2, 3}, + }, + { + in: `[2, 3]`, + ptr: &[...]int{1}, + out: &[...]int{2}, + }, + { + in: `[3]`, + ptr: &[...]int{1, 2}, + out: &[...]int{3, 0}, + }, + } + + for _, tt := range prefillTests { + ptrstr := fmt.Sprintf("%v", tt.ptr) + err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here + if err != nil { + t.Errorf("Unmarshal: %v", err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("Unmarshal(%#q, %s): have %v, want %v", tt.in, ptrstr, tt.ptr, tt.out) + } + } +} + +var invalidUnmarshalTests = []struct { + v interface{} + want string +}{ + {nil, "json: Unmarshal(nil)"}, + {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {(*int)(nil), "json: Unmarshal(nil *int)"}, +} + +func TestInvalidUnmarshal(t *testing.T) { + buf := []byte(`{"a":"1"}`) + for _, tt := range invalidUnmarshalTests { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Errorf("Unmarshal expecting error, got nil") + continue + } + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } + } +} + +var invalidUnmarshalTextTests = []struct { + v interface{} + want string +}{ + {nil, "json: Unmarshal(nil)"}, + {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {(*int)(nil), "json: Unmarshal(nil *int)"}, + {new(net.IP), "json: cannot unmarshal number into Go value of type *net.IP"}, +} + +func TestInvalidUnmarshalText(t *testing.T) { + buf := []byte(`123`) + for _, tt := range invalidUnmarshalTextTests { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Errorf("Unmarshal expecting error, got nil") + continue + } + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } + } +} + +// Test that string option is ignored for invalid types. +// Issue 9812. +func TestInvalidStringOption(t *testing.T) { + num := 0 + item := struct { + T time.Time `json:",string"` + M map[string]string `json:",string"` + S []string `json:",string"` + A [1]string `json:",string"` + I interface{} `json:",string"` + P *int `json:",string"` + }{M: make(map[string]string), S: make([]string, 0), I: num, P: &num} + + data, err := Marshal(item) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + + err = Unmarshal(data, &item) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } +} + +// Test unmarshal behavior with regards to embedded unexported structs. +// +// (Issue 21357) If the embedded struct is a pointer and is unallocated, +// this returns an error because unmarshal cannot set the field. +// +// (Issue 24152) If the embedded struct is given an explicit name, +// ensure that the normal unmarshal logic does not panic in reflect. +// +// (Issue 28145) If the embedded struct is given an explicit name and has +// exported methods, don't cause a panic trying to get its value. +func TestUnmarshalEmbeddedUnexported(t *testing.T) { + type ( + embed1 struct{ Q int } + embed2 struct{ Q int } + embed3 struct { + Q int64 `json:",string"` + } + S1 struct { + *embed1 + R int + } + S2 struct { + *embed1 + Q int + } + S3 struct { + embed1 + R int + } + S4 struct { + *embed1 + embed2 + } + S5 struct { + *embed3 + R int + } + S6 struct { + embed1 `json:"embed1"` + } + S7 struct { + embed1 `json:"embed1"` + embed2 + } + S8 struct { + embed1 `json:"embed1"` + embed2 `json:"embed2"` + Q int + } + S9 struct { + unexportedWithMethods `json:"embed"` + } + ) + + tests := []struct { + in string + ptr interface{} + out interface{} + err error + }{{ + // Error since we cannot set S1.embed1, but still able to set S1.R. + in: `{"R":2,"Q":1}`, + ptr: new(S1), + out: &S1{R: 2}, + err: fmt.Errorf("json: cannot set embedded pointer to unexported struct: json.embed1"), + }, { + // The top level Q field takes precedence. + in: `{"Q":1}`, + ptr: new(S2), + out: &S2{Q: 1}, + }, { + // No issue with non-pointer variant. + in: `{"R":2,"Q":1}`, + ptr: new(S3), + out: &S3{embed1: embed1{Q: 1}, R: 2}, + }, { + // No error since both embedded structs have field R, which annihilate each other. + // Thus, no attempt is made at setting S4.embed1. + in: `{"R":2}`, + ptr: new(S4), + out: new(S4), + }, { + // Error since we cannot set S5.embed1, but still able to set S5.R. + in: `{"R":2,"Q":1}`, + ptr: new(S5), + out: &S5{R: 2}, + err: fmt.Errorf("json: cannot set embedded pointer to unexported struct: json.embed3"), + }, { + // Issue 24152, ensure decodeState.indirect does not panic. + in: `{"embed1": {"Q": 1}}`, + ptr: new(S6), + out: &S6{embed1{1}}, + }, { + // Issue 24153, check that we can still set forwarded fields even in + // the presence of a name conflict. + // + // This relies on obscure behavior of reflect where it is possible + // to set a forwarded exported field on an unexported embedded struct + // even though there is a name conflict, even when it would have been + // impossible to do so according to Go visibility rules. + // Go forbids this because it is ambiguous whether S7.Q refers to + // S7.embed1.Q or S7.embed2.Q. Since embed1 and embed2 are unexported, + // it should be impossible for an external package to set either Q. + // + // It is probably okay for a future reflect change to break this. + in: `{"embed1": {"Q": 1}, "Q": 2}`, + ptr: new(S7), + out: &S7{embed1{1}, embed2{2}}, + }, { + // Issue 24153, similar to the S7 case. + in: `{"embed1": {"Q": 1}, "embed2": {"Q": 2}, "Q": 3}`, + ptr: new(S8), + out: &S8{embed1{1}, embed2{2}, 3}, + }, { + // Issue 228145, similar to the cases above. + in: `{"embed": {}}`, + ptr: new(S9), + out: &S9{}, + }} + + for i, tt := range tests { + err := Unmarshal([]byte(tt.in), tt.ptr) + if !equalError(err, tt.err) { + t.Errorf("#%d: %v, want %v", i, err, tt.err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("#%d: mismatch\ngot: %#+v\nwant: %#+v", i, tt.ptr, tt.out) + } + } +} + +func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) { + tests := []struct { + in string + err error + }{{ + in: `1 false null :`, + err: &SyntaxError{"invalid character ':' looking for beginning of value", 14}, + }, { + in: `1 [] [,]`, + err: &SyntaxError{"invalid character ',' looking for beginning of value", 7}, + }, { + in: `1 [] [true:]`, + err: &SyntaxError{"invalid character ':' after array element", 11}, + }, { + in: `1 {} {"x"=}`, + err: &SyntaxError{"invalid character '=' after object key", 14}, + }, { + in: `falsetruenul#`, + err: &SyntaxError{"invalid character '#' in literal null (expecting 'l')", 13}, + }} + for i, tt := range tests { + dec := NewDecoder(strings.NewReader(tt.in)) + var err error + for { + var v interface{} + if err = dec.Decode(&v); err != nil { + break + } + } + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: got %#v, want %#v", i, err, tt.err) + } + } +} + +type unmarshalPanic struct{} + +func (unmarshalPanic) UnmarshalJSON([]byte) error { panic(0xdead) } + +func TestUnmarshalPanic(t *testing.T) { + defer func() { + if got := recover(); !reflect.DeepEqual(got, 0xdead) { + t.Errorf("panic() = (%T)(%v), want 0xdead", got, got) + } + }() + Unmarshal([]byte("{}"), &unmarshalPanic{}) + t.Fatalf("Unmarshal should have panicked") +} + +// The decoder used to hang if decoding into an interface pointing to its own address. +// See golang.org/issues/31740. +func TestUnmarshalRecursivePointer(t *testing.T) { + var v interface{} + v = &v + data := []byte(`{"a": "b"}`) + + if err := Unmarshal(data, v); err != nil { + t.Fatal(err) + } +} + +type textUnmarshalerString string + +func (m *textUnmarshalerString) UnmarshalText(text []byte) error { + *m = textUnmarshalerString(strings.ToLower(string(text))) + return nil +} + +// Test unmarshal to a map, where the map key is a user defined type. +// See golang.org/issues/34437. +func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) { + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"FOO": "1"}`), &p); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + + if _, ok := p["foo"]; !ok { + t.Errorf(`Key "foo" does not exist in map: %v`, p) + } +} + +func TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) { + // See golang.org/issues/38105. + var p map[textUnmarshalerString]string + if err := Unmarshal([]byte(`{"开源":"12345开源"}`), &p); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if _, ok := p["开源"]; !ok { + t.Errorf(`Key "开源" does not exist in map: %v`, p) + } + + // See golang.org/issues/38126. + type T struct { + F1 string `json:"F1,string"` + } + t1 := T{"aaa\tbbb"} + + b, err := Marshal(t1) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var t2 T + if err := Unmarshal(b, &t2); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + if t1 != t2 { + t.Errorf("Marshal and Unmarshal roundtrip mismatch: want %q got %q", t1, t2) + } + + // See golang.org/issues/39555. + input := map[textUnmarshalerString]string{"FOO": "", `"`: ""} + + encoded, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal unexpected error: %v", err) + } + var got map[textUnmarshalerString]string + if err := Unmarshal(encoded, &got); err != nil { + t.Fatalf("Unmarshal unexpected error: %v", err) + } + want := map[textUnmarshalerString]string{"foo": "", `"`: ""} + if !reflect.DeepEqual(want, got) { + t.Fatalf("Unexpected roundtrip result:\nwant: %q\ngot: %q", want, got) + } +} + +func TestUnmarshalMaxDepth(t *testing.T) { + testcases := []struct { + name string + data string + errMaxDepth bool + }{ + { + name: "ArrayUnderMaxNestingDepth", + data: `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`, + errMaxDepth: false, + }, + { + name: "ArrayOverMaxNestingDepth", + data: `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`, + errMaxDepth: true, + }, + { + name: "ArrayOverStackDepth", + data: `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`, + errMaxDepth: true, + }, + { + name: "ObjectUnderMaxNestingDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`, + errMaxDepth: false, + }, + { + name: "ObjectOverMaxNestingDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`, + errMaxDepth: true, + }, + { + name: "ObjectOverStackDepth", + data: `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`, + errMaxDepth: true, + }, + } + + targets := []struct { + name string + newValue func() interface{} + }{ + { + name: "unstructured", + newValue: func() interface{} { + var v interface{} + return &v + }, + }, + { + name: "typed named field", + newValue: func() interface{} { + v := struct { + A interface{} `json:"a"` + }{} + return &v + }, + }, + { + name: "typed missing field", + newValue: func() interface{} { + v := struct { + B interface{} `json:"b"` + }{} + return &v + }, + }, + { + name: "custom unmarshaler", + newValue: func() interface{} { + v := unmarshaler{} + return &v + }, + }, + } + + for _, tc := range testcases { + for _, target := range targets { + t.Run(target.name+"-"+tc.name, func(t *testing.T) { + err := Unmarshal([]byte(tc.data), target.newValue()) + if !tc.errMaxDepth { + if err != nil { + t.Errorf("unexpected error: %v", err) + } + } else { + if err == nil { + t.Errorf("expected error containing 'exceeded max depth', got none") + } else if !strings.Contains(err.Error(), "exceeded max depth") { + t.Errorf("expected error containing 'exceeded max depth', got: %v", err) + } + } + }) + } + } +} diff --git a/pkg/encoding/json/doc.go b/pkg/encoding/json/doc.go new file mode 100644 index 0000000..cb4449c --- /dev/null +++ b/pkg/encoding/json/doc.go @@ -0,0 +1,19 @@ +package json + +/* + +This package is forked from Go SDK 1.17.2. +Source: https://github.com/golang/go/tree/go1.17.2/src/encoding/json + +Modifications: + +1. Modify JSON marshalling to distinguish between nil and empty values for Arrays, Slices, and Maps. + - Modification lives on L344-347 of encode.go + - This modification allows Go structs with Array/Slice/Map fields with the `omitempty` tag to be serialized + as the empty value when the Go value is empty. Nil values tagged with `omitempty` follow standard behavior + and are not serialized into the resulting output. + - This is needed for performing Kubernetes patch operations using JSON merge semantics so that Array/Slice/Map + fields can be deleted. + +2. Removed `bench_test.go` to avoid bringing in other internal package dependencies. +*/ diff --git a/pkg/encoding/json/encode.go b/pkg/encoding/json/encode.go new file mode 100644 index 0000000..cb4892e --- /dev/null +++ b/pkg/encoding/json/encode.go @@ -0,0 +1,1425 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON as defined in +// RFC 7159. The mapping between JSON and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. If no MarshalJSON method is present but the +// value implements encoding.TextMarshaler instead, Marshal calls +// its MarshalText method and encodes the result as a JSON string. +// The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// So that the JSON will be safe to embed inside HTML