You can access the Merge & Deduplicate Datasets: Remove Duplicates programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in API & Integrations in Apify Console.

Python

JavaScript

CLI

OpenAPI

HTTP

MCP

{
  "openapi": "3.0.1",
  "info": {
    "version": "0.1",
    "x-build-id": "OfoKn04n4kSvY2woR"
  },
  "servers": [
    {
      "url": "https://api.apify.com/v2"
    }
  ],
  "paths": {
    "/acts/grace_hartwell~dataset-merge-dedup/run-sync-get-dataset-items": {
      "post": {
        "operationId": "run-sync-get-dataset-items-grace_hartwell-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    },
    "/acts/grace_hartwell~dataset-merge-dedup/runs": {
      "post": {
        "operationId": "runs-sync-grace_hartwell-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor and returns information about the initiated run in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/runsResponseSchema"
                }
              }
            }
          }
        }
      }
    },
    "/acts/grace_hartwell~dataset-merge-dedup/run-sync": {
      "post": {
        "operationId": "run-sync-grace_hartwell-dataset-merge-dedup",
        "x-openai-isConsequential": false,
        "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.",
        "tags": [
          "Run Actor"
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/inputSchema"
              }
            }
          }
        },
        "parameters": [
          {
            "name": "token",
            "in": "query",
            "required": true,
            "schema": {
              "type": "string"
            },
            "description": "Enter your Apify token here"
          }
        ],
        "responses": {
          "200": {
            "description": "OK"
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "inputSchema": {
        "type": "object",
        "properties": {
          "datasetIds": {
            "title": "Datasets to merge",
            "type": "array",
            "description": "Pick one or more datasets. The picker grants this run READ access to exactly these datasets (limited permissions) – nothing else in your account is touched. Required for real use; if you leave it empty, the Actor runs a small demo on \"Demo items\" instead."
          },
          "dedupFields": {
            "title": "Deduplicate by fields",
            "type": "array",
            "description": "Fields whose combined value must be unique, e.g. [\"url\"] or [\"product.id\", \"shop\"]. Dot paths for nested fields. Leave empty to remove only fully identical items.",
            "items": {
              "type": "string"
            }
          },
          "caseInsensitive": {
            "title": "Case-insensitive comparison",
            "type": "boolean",
            "description": "Treat \"ABC\" and \"abc\" as equal.",
            "default": false
          },
          "trimStrings": {
            "title": "Trim whitespace",
            "type": "boolean",
            "description": "Ignore leading/trailing whitespace when comparing strings.",
            "default": true
          },
          "missingFieldMode": {
            "title": "Items missing all dedup fields",
            "enum": [
              "keep-all",
              "as-empty",
              "drop"
            ],
            "type": "string",
            "description": "keep-all: output them (never treated as duplicates). as-empty: treat missing as an empty value (all such items collapse into one). drop: skip them.",
            "default": "keep-all"
          },
          "onlyNewItems": {
            "title": "Only output items not seen in previous runs",
            "type": "boolean",
            "description": "Remembers item keys in a key-value store created by this Actor (named dedup-state-<state key>) and skips items already output in earlier runs. Great for scheduled scrapers.",
            "default": false
          },
          "stateKey": {
            "title": "State key",
            "type": "string",
            "description": "Name of the remembered state. Use different keys for unrelated pipelines.",
            "default": "default"
          },
          "updateState": {
            "title": "Update state after run",
            "type": "boolean",
            "description": "Turn off for a dry run that compares against the state without changing it.",
            "default": true
          },
          "referenceDatasetIds": {
            "title": "Reference datasets (filter only)",
            "type": "array",
            "description": "Optional: items whose key appears in these datasets are treated as already seen and not output. READ access only."
          },
          "outputFields": {
            "title": "Output only these fields",
            "type": "array",
            "description": "Optional projection, e.g. [\"url\", \"title\", \"price.value\"]. Empty = full items.",
            "items": {
              "type": "string"
            }
          },
          "addSourceDatasetId": {
            "title": "Add _sourceDatasetId",
            "type": "boolean",
            "description": "Add the ID of the dataset each item came from.",
            "default": false
          },
          "maxOutputItems": {
            "title": "Max output items",
            "minimum": 0,
            "type": "integer",
            "description": "0 = unlimited. Output is also capped by your 'Max total charge' budget.",
            "default": 0
          },
          "pageSize": {
            "title": "Read page size",
            "minimum": 100,
            "maximum": 50000,
            "type": "integer",
            "description": "Items read per API call (streaming). Lower it for very large items.",
            "default": 5000
          },
          "pushBatchSize": {
            "title": "Write batch size",
            "minimum": 1,
            "maximum": 5000,
            "type": "integer",
            "description": "Items written per API call.",
            "default": 1000
          },
          "demoItems": {
            "title": "Demo items (used only when no dataset is selected)",
            "type": "array",
            "description": "A tiny built-in example so a run without datasets still shows how deduplication works (e.g. Apify's default-input test). Ignored as soon as you select at least one dataset above. Max 100 items. Output items are charged like normal output items.",
            "default": [
              {
                "url": "https://example.com/product/1",
                "title": "Blue mug",
                "price": 12.9
              },
              {
                "url": "https://example.com/product/2",
                "title": "Red mug",
                "price": 11.5
              },
              {
                "url": "https://example.com/product/1",
                "title": "Blue mug (duplicate)",
                "price": 12.9
              },
              {
                "url": " https://example.com/product/3 ",
                "title": "Green mug",
                "price": 13
              },
              {
                "url": "https://example.com/product/2",
                "title": "Red mug (duplicate)",
                "price": 11.5
              }
            ]
          }
        }
      },
      "runsResponseSchema": {
        "type": "object",
        "properties": {
          "data": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "actId": {
                "type": "string"
              },
              "userId": {
                "type": "string"
              },
              "startedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "finishedAt": {
                "type": "string",
                "format": "date-time",
                "example": "2025-01-08T00:00:00.000Z"
              },
              "status": {
                "type": "string",
                "example": "READY"
              },
              "meta": {
                "type": "object",
                "properties": {
                  "origin": {
                    "type": "string",
                    "example": "API"
                  },
                  "userAgent": {
                    "type": "string"
                  }
                }
              },
              "stats": {
                "type": "object",
                "properties": {
                  "inputBodyLen": {
                    "type": "integer",
                    "example": 2000
                  },
                  "rebootCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "restartCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "resurrectCount": {
                    "type": "integer",
                    "example": 0
                  },
                  "computeUnits": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "options": {
                "type": "object",
                "properties": {
                  "build": {
                    "type": "string",
                    "example": "latest"
                  },
                  "timeoutSecs": {
                    "type": "integer",
                    "example": 300
                  },
                  "memoryMbytes": {
                    "type": "integer",
                    "example": 1024
                  },
                  "diskMbytes": {
                    "type": "integer",
                    "example": 2048
                  }
                }
              },
              "buildId": {
                "type": "string"
              },
              "defaultKeyValueStoreId": {
                "type": "string"
              },
              "defaultDatasetId": {
                "type": "string"
              },
              "defaultRequestQueueId": {
                "type": "string"
              },
              "buildNumber": {
                "type": "string",
                "example": "1.0.0"
              },
              "containerUrl": {
                "type": "string"
              },
              "usage": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "integer",
                    "example": 1
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              },
              "usageTotalUsd": {
                "type": "number",
                "example": 0.00005
              },
              "usageUsd": {
                "type": "object",
                "properties": {
                  "ACTOR_COMPUTE_UNITS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATASET_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "KEY_VALUE_STORE_WRITES": {
                    "type": "number",
                    "example": 0.00005
                  },
                  "KEY_VALUE_STORE_LISTS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_READS": {
                    "type": "integer",
                    "example": 0
                  },
                  "REQUEST_QUEUE_WRITES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_INTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "DATA_TRANSFER_EXTERNAL_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_RESIDENTIAL_TRANSFER_GBYTES": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_SERPS": {
                    "type": "integer",
                    "example": 0
                  },
                  "PROXY_UNBLOCKER_UNITS": {
                    "type": "integer",
                    "example": 0
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

Merge & Deduplicate Datasets: Remove Duplicates OpenAPI definition

OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.

OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.

By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.

You can download the OpenAPI definitions for Merge & Deduplicate Datasets: Remove Duplicates from the options below:

OpenAPI.json

If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.

You can also check out our other API clients:

Merge & Deduplicate Datasets: Remove Duplicates API in Python

Merge & Deduplicate Datasets: Remove Duplicates API in JavaScript

Merge & Deduplicate Datasets: Remove Duplicates API through CLI

Merge & Deduplicate Datasets: Remove Duplicates API

Instagram Scraper

apify/instagram-scraper

Extract Instagram posts, reels, profiles, places, hashtags, carousels, and comments. Get data from Instagram using one or more Instagram URLs or search queries: content, context, metrics, metadata. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Apify

449K

4.7

(603)

Google Maps Scraper

compass/crawler-google-places

Extract data from thousands of Google Maps locations and businesses, including reviews, reviewer details, images, contact info, including full name, email, and job title, opening hours, prices & more. Export data, run via API, schedule and monitor runs, or integrate with other tools.

Compass

667K

4.7

(1,818)

TikTok Scraper

clockworks/tiktok-scraper

Extract data from TikTok videos, hashtags, and users. Use URLs or search queries to scrape TikTok profiles, hashtags, posts, URLs, shares, followers, hearts, names, video, and music-related data. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Clockworks

362K

4.7

(370)

Google Search Results Scraper

apify/google-search-scraper

Scrape Google Search Engine Results Pages (SERPs). Select the country or language and extract organic and paid results, AI Mode, AI overviews, ads, queries, People Also Ask, prices, reviews, like a Google SERP API. Export data, run the scraper via API, schedule runs, or integrate with other tools.

Apify

218K

4.6

(220)

Instagram Profile Scraper

apify/instagram-profile-scraper

Scrape all Instagram profile info. Just add Instagram usernames, IDs or URLs and extract name, join date, number of followers, location, bio, website, related profiles, video&post count, latest posts. Export scraped data, schedule scraper via API, and integrate with other tools or AI workflows.

Apify

264K

4.7

(175)

Company Employees Scraper for LinkedIn | No Cookies

apimaestro/linkedin-company-employees-scraper-no-cookies

Extract LinkedIn company employees without sharing your cookies or account. Get structured data including profile details, job titles, and current positions. No login required.

API Maestro

10K

2.9

(18)

LinkedIn Profile Scraper + Email (No Cookies)

harvestapi/linkedin-profile-scraper

Extract detailed information from LinkedIn profiles in bulk, including complete work experience, education history, skills and more. No cookies or account required.

HarvestAPI

87K

4.6

(93)

Tweet Scraper V2 - X Scraper / Twitter Scraper / X.com Scraper

apidojo/tweet-scraper

Lightning-fast search, URL, list, and profile scraping, with customizable filters. At $0.40 per 1000 tweets, and 30-80 tweets per second, it is ideal for researchers, entrepreneurs, and businesses! Get comprehensive insights from Twitter (X) now!

API Dojo

119K

3.9

(196)

Instagram Profile Scraper (Bio & Posts)

coderx/instagram-profile-scraper-bio-posts

Instagram Profile Insights Scraper is a lightweight tool that collects detailed public data from a list of Instagram usernames. It fetches profile info and recent posts to help you analyze accounts at scale.

CoderX

9.8K

5.0

(9)

Instagram Post Scraper

apify/instagram-post-scraper

Scrape Instagram posts. Just add one or more Instagram usernames and get your data in seconds including caption, metrics, images, mentions, coauthors, recent comments, sponsored status, video duration, views. Export scraped data, schedule scraper via API, integrate with other tools or AI workflows.

Apify

148K

4.3

(138)