Semantic RAG Document Chunker and Token Slicer

A high-performance document segmenter designed for Retrieval Augmented Generation (RAG) and vector databases. Uses exact token counting algorithms to slice large text, Markdown, or HTML files into context-aware chunks without breaking sentences, Markdown tables, or code blocks in half.

---

Why Use This Actor?

Standard character-length chunkers blindly cut paragraphs and code blocks mid-sentence, causing retrieval distortion and embedding corruption. This tool provides:

• Exact token budgeting based on cl100k_base (OpenAI / Anthropic compatible).
• Semantic boundary preservation (code fences, markdown headings, list items).
• Heading breadcrumb propagation so each chunk retains its parent outline context.
• Configurable overlap token windows for smooth context transitions.

---

Dual Output Envelope (Agent Native)

Every record contains both structured chunks with metadata and a concise markdown report:

{
  "id": "doc-001",
  "success": true,
  "meta": {
    "processing_time_ms": 1.8,
    "items_count": 2
  },
  "data": {
    "document_id": "doc-001",
    "total_chunks": 2,
    "total_tokens": 140,
    "chunks": [
      {
        "chunk_index": 0,
        "content": "# Introduction to Autonomous RAG\nRetrieval Augmented Generation combines information retrieval with language modeling.",
        "token_count": 75,
        "headings": ["# Introduction to Autonomous RAG"]
      }
    ]
  },
  "markdown_summary": "### Document Chunking Summary\n- Document ID: doc-001\n- Chunks Generated: 2\n- Total Tokens: 140\n- Latency: 1.8ms\n"
}

---

Input Schema

Field	Type	Default	Description
items	Array of Objects	Sample Document	Array of objects with id and text to segment into chunks.
max_tokens_per_chunk	Integer	200	Maximum token limit per chunk.
chunk_overlap_tokens	Integer	30	Overlap window size between adjacent chunks.

---

Pay-Per-Event (PPE) Pricing

• Primary Event: document-chunked
• Cost: $0.001 per document chunked ($1.00 per 1,000 documents).
• Initial 5 seconds of compute are waived via apify-actor-start.

---

Integration Code in 10+ Languages

cURL

curl -X POST "https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"items": [{"id": "doc1", "text": "# Header\nLong text..."}], "max_tokens_per_chunk": 250}'

Python

from apify_client import ApifyClient

client = ApifyClient("YOUR_API_TOKEN")
run = client.actor("n4d.h1dr/rag-smart-chunker").call(run_input={
    "items": [{"id": "doc1", "text": "Article content..."}],
    "max_tokens_per_chunk": 300
})
for item in client.dataset(run["defaultDatasetId"]).iterate_items():
    print(f"Chunks: {item['data']['total_chunks']}")

Node.js / TypeScript

import { ApifyClient } from 'apify-client';

const client = new ApifyClient({ token: 'YOUR_API_TOKEN' });
const run = await client.actor('n4d.h1dr/rag-smart-chunker').call({
  items: [{ id: 'doc1', text: 'Text to split...' }]
});
const { items } = await client.dataset(run.defaultDatasetId).listItems();
console.log(items[0].data.chunks);

Go

package main
import (
    "bytes"
    "net/http"
)
func main() {
    url := "https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN"
    payload := []byte(`{"items": [{"id": "doc1", "text": "Content..."}]}`)
    http.Post(url, "application/json", bytes.NewBuffer(payload))
}

Ruby

require 'net/http'
require 'json'
uri = URI('https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN')
req = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
req.body = { items: [{ id: 'doc1', text: 'Content to chunk' }] }.to_json
res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(req) }

PHP

<?php
$ch = curl_init('https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN');
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode([
    'items' => [['id' => '1', 'text' => 'Long text to slice']]
]));
curl_setopt($ch, CURLOPT_HTTPHEADER, ['Content-Type: application/json']);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$res = curl_exec($ch);

Java

import java.net.URI;
import java.net.http.*;
public class Main {
    public static void main(String[] args) throws Exception {
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request = HttpRequest.newBuilder()
            .uri(URI.create("https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN"))
            .header("Content-Type", "application/json")
            .POST(HttpRequest.BodyPublishers.ofString("{\"items\": [{\"id\": \"1\", \"text\": \"Document...\"}]}"))
            .build();
        client.send(request, HttpResponse.BodyHandlers.ofString());
    }
}

C# (.NET)

using System.Net.Http.Json;
var client = new HttpClient();
var payload = new { items = new[] { new { id = "1", text = "Document to chunk" } } };
await client.PostAsJsonAsync("https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN", payload);

Rust

use reqwest::Client;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client.post("https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN")
        .json(&serde_json::json!({"items": [{"id": "1", "text": "Document..."}]}))
        .send().await?;
    Ok(())
}

Swift

import Foundation
var request = URLRequest(url: URL(string: "https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN")!)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
request.httpBody = try? JSONSerialization.data(withJSONObject: ["items": [["id": "1", "text": "RAG text"]]])
URLSession.shared.dataTask(with: request).resume()

n8n / Make.com Webhook Node

Add an HTTP Request node in n8n or Make:

• Method: POST
• URL: https://api.apify.com/v2/acts/n4d.h1dr~rag-smart-chunker/runs?token=YOUR_API_TOKEN
• Body Parameters: { "items": [ { "id": "={{ $json.id }}", "text": "={{ $json.raw_doc }}" } ], "max_tokens_per_chunk": 300 }