Merge, Dedup & Transform Datasets - Limited permissions
Pricing
Pay per usage
Limited permission version of the popular Actor. The ultimate dataset processor. Extremely fast merging, deduplications & transformations all in a single run.
You can access the Merge, Dedup & Transform Datasets - Limited permissions programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in API & Integrations in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "0.0", "x-build-id": "1u7rA5G914c4eYTbZ" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/lukaskrivka~dedup-datasets-limited-permissions/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-lukaskrivka-dedup-datasets-limited-permissions", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/lukaskrivka~dedup-datasets-limited-permissions/runs": { "post": { "operationId": "runs-sync-lukaskrivka-dedup-datasets-limited-permissions", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/lukaskrivka~dedup-datasets-limited-permissions/run-sync": { "post": { "operationId": "run-sync-lukaskrivka-dedup-datasets-limited-permissions", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "datasetIds": { "title": "Dataset IDs", "type": "array", "description": "Datasets that should be deduplicated and merged", "items": { "type": "string" } }, "fields": { "title": "Fields for deduplication", "type": "array", "description": "Fields whose combination should be unique for the item to be considered unique. If none are provided, the actor does not perform deduplication.", "items": { "type": "string" } }, "output": { "title": "What to output", "enum": [ "unique-items", "duplicate-items", "nothing" ], "type": "string", "description": "What will be pushed to the dataset from this actor", "default": "unique-items" }, "mode": { "title": "Mode", "enum": [ "dedup-after-load", "dedup-as-loading" ], "type": "string", "description": "How the loading and deduplication process will work.", "default": "dedup-after-load" }, "outputDatasetId": { "title": "Output dataset ID or name (optional)", "type": "string", "description": "Optionally can push into dataset of your choice. If you provide a dataset name that doesn't exist, a new named dataset will be created." }, "fieldsToLoad": { "title": "Limit fields to load", "type": "array", "description": "You can choose which fields to load only. Useful to speed up the loading and reduce memory needs.", "items": { "type": "string" } }, "preDedupTransformFunction": { "title": "Pre dedup transform function", "type": "string", "description": "Function to transform items before deduplication is applied. For 'dedup-after-load' mode this is done for all items at once. For 'dedup-as-loading' this is applied to each batch separately." }, "postDedupTransformFunction": { "title": "Post dedup transform function", "type": "string", "description": "Function to transform items after deduplication is applied. For 'dedup-after-load' mode this is done for all items at once. For 'dedup-as-loading' this is applied to each batch separately." }, "outputTo": { "title": "Where to output", "enum": [ "dataset", "key-value-store" ], "type": "string", "description": "Either can output to a single dataset or to split data into KV records depending on upload batch size. KV is upload is much faster but data end up in many files.", "default": "dataset" }, "parallelLoads": { "title": "Parallel loads", "maximum": 100, "type": "integer", "description": "Datasets can be loaded in parallel batches to speed things up if needed.", "default": 10 }, "parallelPushes": { "title": "Parallel pushes", "minimum": 1, "maximum": 50, "type": "integer", "description": "Deduped data can be pushed in parallel batches to speed things up if needed. If you want the data to be in the exact same order, you need to set this to 1.", "default": 5 }, "uploadBatchSize": { "title": "Upload batch size", "minimum": 10, "maximum": 1000, "type": "integer", "description": "How many items it should upload in one pushData call. Useful to not overload Apify API. Only important for dataset upload.", "default": 500 }, "batchSizeLoad": { "title": "Download batch size", "type": "integer", "description": "How many items it will load in a single batch.", "default": 50000 }, "offset": { "title": "Offset (how many items to skip from start)", "type": "integer", "description": "By default we don't skip any items which is the same as setting offset to 0. For multiple datasets, it takes offset into the sum of their item counts but that is not very useful." }, "limit": { "title": "Limit (how many items to load)", "type": "integer", "description": "By default we don't limit the number loaded items" }, "verboseLog": { "title": "verbose log", "type": "boolean", "description": "Good for smaller runs. Large runs might run out of log space.", "default": false }, "nullAsUnique": { "title": "Null fields are unique", "type": "boolean", "description": "If you want to treat null (or missing) fields as always unique items.", "default": false }, "datasetIdsOfFilterItems": { "title": "Dataset IDs for just deduping", "type": "array", "description": "The items from these datasets will be just used as a dedup filter for the main datasets. These items are loaded first and then the main datasets are compared for uniqueness and pushed.", "items": { "type": "string" } }, "customInputData": { "title": "Custom input data", "type": "object", "description": "You can pass custom data as a JSON object to be accessible in the transform functions as part of the 2nd parameter object." }, "appendDatasetIds": { "title": "Append dataset IDs to items", "type": "boolean", "description": "Useful for transform functions. Each item will get a field `__datasetId__` with the dataset ID it came from.", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 }, "PROXY_UNBLOCKER_UNITS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Merge, Dedup & Transform Datasets - Limited permissions from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: