From 2cd61aa6201826193aecaab6fda874822989e63a Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Sat, 1 Jun 2024 19:52:07 +0300 Subject: [PATCH] Implement document PATCH operation --- .gitignore | 4 + README.md | 46 ++++++----- api/handlers/documents.go | 79 +++++++++++++++++++ api/handlers/middleware/authentication.go | 71 ++++++++++------- api/router.go | 1 + api/tests/documents_test.go | 72 ++++++++++++++++- go.mod | 2 + go.sum | 4 + .../authentication/authentication_test.go | 5 ++ 9 files changed, 233 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index 92d5d2e..b1b6a25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ dist/ ignored/ +explorer_www/ +main +save.json +.vscode/ diff --git a/README.md b/README.md index a886852..9f4b7ea 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # Cosmium -Cosmium is a lightweight Cosmos DB emulator designed to facilitate local development and testing. While it aims to provide developers with a solution for running a local database during development, it's important to note that it's not 100% compatible with Cosmos DB. However, it serves as a convenient tool for E2E or integration tests during the CI/CD pipeline. Read more about compatibility [here](docs/compatibility.md). +Cosmium is a lightweight Cosmos DB emulator designed to facilitate local development and testing. While it aims to provide developers with a solution for running a local database during development, it's important to note that it's not 100% compatible with Cosmos DB. However, it serves as a convenient tool for E2E or integration tests during the CI/CD pipeline. Read more about compatibility [here](./docs/compatibility.md). One of Cosmium's notable features is its ability to save and load state to a single JSON file. This feature makes it easy to load different test cases or share state with other developers, enhancing collaboration and efficiency in development workflows. # Getting Started + ### Installation via Homebrew + You can install Cosmium using Homebrew by adding the `pikami/brew` tap and then installing the package. ```sh @@ -23,10 +25,10 @@ You can download the latest version of Cosmium from the [GitHub Releases page](h Cosmium is available for the following platforms: -* **Linux**: cosmium-linux-amd64 -* **macOS**: cosmium-darwin-amd64 -* **macOS on Apple Silicon**: cosmium-darwin-arm64 -* **Windows**: cosmium-windows-amd64.exe +- **Linux**: cosmium-linux-amd64 +- **macOS**: cosmium-darwin-amd64 +- **macOS on Apple Silicon**: cosmium-darwin-arm64 +- **Windows**: cosmium-windows-amd64.exe ### Running Cosmium @@ -37,11 +39,12 @@ cosmium -Persist "./save.json" ``` Connection String Example: + ``` AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==; ``` -### Running Cosmos DB Explorer +### Running Cosmos DB Explorer If you want to run Cosmos DB Explorer alongside Cosmium, you'll need to build it yourself and point the `-ExplorerDir` argument to the dist directory. Please refer to the [Cosmos DB Explorer repository](https://github.com/Azure/cosmos-explorer) for instructions on building the application. @@ -50,6 +53,7 @@ Once running, the explorer can be reached by navigating following URL: `https:// ### Running with docker (optional) If you wan to run the application using docker, configure it using environment variables see example: + ```sh docker run --rm \ -e Persist=/save.json \ @@ -66,24 +70,26 @@ To disable SSL and run Cosmium on HTTP instead, you can use the `-DisableTls` fl ### Other Available Arguments -* **-AccountKey**: Account key for authentication (default "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==") -* **-DisableAuth**: Disable authentication -* **-Host**: Hostname (default "localhost") -* **-InitialData**: Path to JSON containing initial state -* **-Persist**: Saves data to the given path on application exit (When `-InitialData` argument is not supplied, it will try to load data from path supplied in `-Persist`) -* **-Port**: Listen port (default 8081) -* **-Debug**: Runs application in debug mode, this provides additional logging +- **-AccountKey**: Account key for authentication (default "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==") +- **-DisableAuth**: Disable authentication +- **-Host**: Hostname (default "localhost") +- **-InitialData**: Path to JSON containing initial state +- **-Persist**: Saves data to the given path on application exit (When `-InitialData` argument is not supplied, it will try to load data from path supplied in `-Persist`) +- **-Port**: Listen port (default 8081) +- **-Debug**: Runs application in debug mode, this provides additional logging These arguments allow you to configure various aspects of Cosmium's behavior according to your requirements. All mentioned arguments can also be set using environment variables: -* **COSMIUM_ACCOUNTKEY** for `-AccountKey` -* **COSMIUM_DISABLEAUTH** for `-DisableAuth` -* **COSMIUM_HOST** for `-Host` -* **COSMIUM_INITIALDATA** for `-InitialData` -* **COSMIUM_PERSIST** for `-Persist` -* **COSMIUM_PORT** for `-Port` -* **COSMIUM_DEBUG** for `-Debug` + +- **COSMIUM_ACCOUNTKEY** for `-AccountKey` +- **COSMIUM_DISABLEAUTH** for `-DisableAuth` +- **COSMIUM_HOST** for `-Host` +- **COSMIUM_INITIALDATA** for `-InitialData` +- **COSMIUM_PERSIST** for `-Persist` +- **COSMIUM_PORT** for `-Port` +- **COSMIUM_DEBUG** for `-Debug` # License + This project is [MIT licensed](./LICENSE). diff --git a/api/handlers/documents.go b/api/handlers/documents.go index e5bb7a4..a68e2c8 100644 --- a/api/handlers/documents.go +++ b/api/handlers/documents.go @@ -1,11 +1,14 @@ package handlers import ( + "encoding/json" "fmt" "net/http" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gin-gonic/gin" "github.com/pikami/cosmium/internal/constants" + "github.com/pikami/cosmium/internal/logger" "github.com/pikami/cosmium/internal/repositories" repositorymodels "github.com/pikami/cosmium/internal/repository_models" ) @@ -100,6 +103,82 @@ func ReplaceDocument(c *gin.Context) { c.IndentedJSON(http.StatusInternalServerError, gin.H{"message": "Unknown error"}) } +func PatchDocument(c *gin.Context) { + databaseId := c.Param("databaseId") + collectionId := c.Param("collId") + documentId := c.Param("docId") + + document, status := repositories.GetDocument(databaseId, collectionId, documentId) + if status == repositorymodels.StatusNotFound { + c.IndentedJSON(http.StatusNotFound, gin.H{"message": "NotFound"}) + return + } + + var requestBody map[string]interface{} + if err := c.BindJSON(&requestBody); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()}) + return + } + + operations := requestBody["operations"] + operationsBytes, err := json.Marshal(operations) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"message": "Could not decode operations"}) + return + } + + patch, err := jsonpatch.DecodePatch(operationsBytes) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()}) + return + } + + currentDocumentBytes, err := json.Marshal(document) + if err != nil { + logger.Error("Failed to marshal existing document:", err) + c.JSON(http.StatusInternalServerError, gin.H{"message": "Failed to marshal existing document"}) + return + } + + modifiedDocumentBytes, err := patch.Apply(currentDocumentBytes) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()}) + return + } + + var modifiedDocument map[string]interface{} + err = json.Unmarshal(modifiedDocumentBytes, &modifiedDocument) + if err != nil { + logger.Error("Failed to unmarshal modified document:", err) + c.JSON(http.StatusInternalServerError, gin.H{"message": "Failed to unmarshal modified document"}) + return + } + + if modifiedDocument["id"] != document["id"] { + c.JSON(http.StatusUnprocessableEntity, gin.H{"message": "The ID field cannot be modified"}) + return + } + + status = repositories.DeleteDocument(databaseId, collectionId, documentId) + if status == repositorymodels.StatusNotFound { + c.IndentedJSON(http.StatusNotFound, gin.H{"message": "NotFound"}) + return + } + + createdDocument, status := repositories.CreateDocument(databaseId, collectionId, modifiedDocument) + if status == repositorymodels.Conflict { + c.IndentedJSON(http.StatusConflict, gin.H{"message": "Conflict"}) + return + } + + if status == repositorymodels.StatusOk { + c.IndentedJSON(http.StatusCreated, createdDocument) + return + } + + c.IndentedJSON(http.StatusInternalServerError, gin.H{"message": "Unknown error"}) +} + func DocumentsPost(c *gin.Context) { databaseId := c.Param("databaseId") collectionId := c.Param("collId") diff --git a/api/handlers/middleware/authentication.go b/api/handlers/middleware/authentication.go index ee3d6e9..f8965c2 100644 --- a/api/handlers/middleware/authentication.go +++ b/api/handlers/middleware/authentication.go @@ -19,35 +19,8 @@ func Authentication() gin.HandlerFunc { return } - var resourceType string - parts := strings.Split(requestUrl, "/") - switch len(parts) { - case 2, 3: - resourceType = parts[1] - case 4, 5: - resourceType = parts[3] - case 6, 7: - resourceType = parts[5] - } - - databaseId, _ := c.Params.Get("databaseId") - collId, _ := c.Params.Get("collId") - docId, _ := c.Params.Get("docId") - var resourceId string - if databaseId != "" { - resourceId += "dbs/" + databaseId - } - if collId != "" { - resourceId += "/colls/" + collId - } - if docId != "" { - resourceId += "/docs/" + docId - } - - isFeed := c.Request.Header.Get("A-Im") == "Incremental Feed" - if resourceType == "pkranges" && isFeed { - resourceId = collId - } + resourceType := urlToResourceType(requestUrl) + resourceId := requestToResourceId(c) authHeader := c.Request.Header.Get("authorization") date := c.Request.Header.Get("x-ms-date") @@ -67,3 +40,43 @@ func Authentication() gin.HandlerFunc { } } } + +func urlToResourceType(requestUrl string) string { + var resourceType string + parts := strings.Split(requestUrl, "/") + switch len(parts) { + case 2, 3: + resourceType = parts[1] + case 4, 5: + resourceType = parts[3] + case 6, 7: + resourceType = parts[5] + } + + return resourceType +} + +func requestToResourceId(c *gin.Context) string { + databaseId, _ := c.Params.Get("databaseId") + collId, _ := c.Params.Get("collId") + docId, _ := c.Params.Get("docId") + resourceType := urlToResourceType(c.Request.URL.String()) + + var resourceId string + if databaseId != "" { + resourceId += "dbs/" + databaseId + } + if collId != "" { + resourceId += "/colls/" + collId + } + if docId != "" { + resourceId += "/docs/" + docId + } + + isFeed := c.Request.Header.Get("A-Im") == "Incremental Feed" + if resourceType == "pkranges" && isFeed { + resourceId = collId + } + + return resourceId +} diff --git a/api/router.go b/api/router.go index 81e0ded..9f1fc76 100644 --- a/api/router.go +++ b/api/router.go @@ -27,6 +27,7 @@ func CreateRouter() *gin.Engine { router.GET("/dbs/:databaseId/colls/:collId/docs", handlers.GetAllDocuments) router.GET("/dbs/:databaseId/colls/:collId/docs/:docId", handlers.GetDocument) router.PUT("/dbs/:databaseId/colls/:collId/docs/:docId", handlers.ReplaceDocument) + router.PATCH("/dbs/:databaseId/colls/:collId/docs/:docId", handlers.PatchDocument) router.DELETE("/dbs/:databaseId/colls/:collId/docs/:docId", handlers.DeleteDocument) router.POST("/dbs/:databaseId/colls", handlers.CreateCollection) diff --git a/api/tests/documents_test.go b/api/tests/documents_test.go index 10bf5cb..393543d 100644 --- a/api/tests/documents_test.go +++ b/api/tests/documents_test.go @@ -3,10 +3,14 @@ package tests_test import ( "context" "encoding/json" + "errors" "fmt" + "net/http" + "net/http/httptest" "reflect" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" "github.com/pikami/cosmium/api/config" "github.com/pikami/cosmium/internal/repositories" @@ -49,7 +53,7 @@ func testCosmosQuery(t *testing.T, } } -func Test_Documents(t *testing.T) { +func documents_InitializeDb(t *testing.T) (*httptest.Server, *azcosmos.ContainerClient) { repositories.CreateDatabase(repositorymodels.Database{ID: testDatabaseName}) repositories.CreateCollection(testDatabaseName, repositorymodels.Collection{ ID: testCollectionName, @@ -65,7 +69,6 @@ func Test_Documents(t *testing.T) { repositories.CreateDocument(testDatabaseName, testCollectionName, map[string]interface{}{"id": "67890", "pk": "456", "isCool": true}) ts := runTestServer() - defer ts.Close() client, err := azcosmos.NewClientFromConnectionString( fmt.Sprintf("AccountEndpoint=%s;AccountKey=%s", ts.URL, config.Config.AccountKey), @@ -76,6 +79,13 @@ func Test_Documents(t *testing.T) { collectionClient, err := client.NewContainer(testDatabaseName, testCollectionName) assert.Nil(t, err) + return ts, collectionClient +} + +func Test_Documents(t *testing.T) { + ts, collectionClient := documents_InitializeDb(t) + defer ts.Close() + t.Run("Should query document", func(t *testing.T) { testCosmosQuery(t, collectionClient, "SELECT c.id, c[\"pk\"] FROM c ORDER BY c.id", @@ -137,3 +147,61 @@ func Test_Documents(t *testing.T) { ) }) } + +func Test_Documents_Patch(t *testing.T) { + ts, collectionClient := documents_InitializeDb(t) + defer ts.Close() + + t.Run("Should PATCH document", func(t *testing.T) { + context := context.TODO() + expectedData := map[string]interface{}{"id": "67890", "pk": "456", "newField": "newValue"} + + patch := azcosmos.PatchOperations{} + patch.AppendAdd("/newField", "newValue") + patch.AppendRemove("/isCool") + + itemResponse, err := collectionClient.PatchItem( + context, + azcosmos.PartitionKey{}, + "67890", + patch, + &azcosmos.ItemOptions{ + EnableContentResponseOnWrite: false, + }, + ) + assert.Nil(t, err) + + var itemResponseBody map[string]string + json.Unmarshal(itemResponse.Value, &itemResponseBody) + + assert.Equal(t, expectedData["id"], itemResponseBody["id"]) + assert.Equal(t, expectedData["pk"], itemResponseBody["pk"]) + assert.Empty(t, itemResponseBody["isCool"]) + assert.Equal(t, expectedData["newField"], itemResponseBody["newField"]) + }) + + t.Run("Should not allow to PATCH document ID", func(t *testing.T) { + context := context.TODO() + + patch := azcosmos.PatchOperations{} + patch.AppendReplace("/id", "newValue") + + _, err := collectionClient.PatchItem( + context, + azcosmos.PartitionKey{}, + "67890", + patch, + &azcosmos.ItemOptions{ + EnableContentResponseOnWrite: false, + }, + ) + assert.NotNil(t, err) + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + assert.Equal(t, http.StatusUnprocessableEntity, respErr.StatusCode) + } else { + panic(err) + } + }) +} diff --git a/go.mod b/go.mod index b5aafdd..029345b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.6 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v0.3.6 + github.com/evanphx/json-patch/v5 v5.9.0 github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.1.1 github.com/stretchr/testify v1.8.4 @@ -30,6 +31,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect diff --git a/go.sum b/go.sum index 0eb52e1..52a2ffe 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583j github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -64,6 +66,8 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/internal/authentication/authentication_test.go b/internal/authentication/authentication_test.go index 20c2f60..01c7157 100644 --- a/internal/authentication/authentication_test.go +++ b/internal/authentication/authentication_test.go @@ -32,4 +32,9 @@ func Test_GenerateSignature(t *testing.T) { signature := authentication.GenerateSignature("GET", "pkranges", "m4d+xG08uVM=", testDate, config.DefaultAccountKey) assert.Equal(t, "6S5ceZsl2EXWB3Jo5bJcK7zv8NxXnsxWPWD9TH3nNMo=", signature) }) + + t.Run("Should generate PATCH signature", func(t *testing.T) { + signature := authentication.GenerateSignature("PATCH", "docs", "dbs/test-db/colls/test-coll/docs/67890", testDate, config.DefaultAccountKey) + assert.Equal(t, "VR1ddfxKBXnoaT+b3WkhyYVc9JmGNpTnaRmyDM44398=", signature) + }) }