Under maintenance

Pricing

$0.00005 / actor start

Try for free

Go to Apify Store

depracted (to be removed)

Under maintenance

Try for free

Pricing

$0.00005 / actor start

Rating

0.0

(0)

Developer

Asad Ali

Actor stats

Bookmarked

Total users

Monthly active users

21 days ago

Last modified

Dockerfile

FROM golang:1.23-alpine AS builder
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /server .

FROM alpine:3.20
RUN apk --no-cache add ca-certificates
COPY --from=builder /server /server
EXPOSE 8080
ENTRYPOINT ["/server"]

go.mod

module social-api

go 1.23

go.sum

main.go

package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"

	"social-api/server"
)

func main() {
	// Apify Standby mode sets APIFY_ACTOR_STANDBY_PORT; fall back to PORT or 8080
	port := os.Getenv("APIFY_ACTOR_STANDBY_PORT")
	if port == "" {
		port = os.Getenv("PORT")
	}
	if port == "" {
		port = "8080"
	}

	srv := server.New()

	// Graceful shutdown
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		log.Printf("Starting server on :%s", port)
		if err := srv.Start(":" + port); err != nil {
			log.Fatalf("Server error: %v", err)
		}
	}()

	<-quit
	log.Println("Shutting down server...")
	srv.Shutdown()
}

.actor/Dockerfile

FROM golang:1.23-alpine AS builder
WORKDIR /app
COPY go.mod ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /server .

FROM alpine:3.20
RUN apk --no-cache add ca-certificates
COPY --from=builder /server /server
COPY config/proxies.json /app/config/proxies.json

ENV PORT=8080
EXPOSE 8080

ENTRYPOINT ["/server"]

.actor/actor.json

{
    "actorSpecification": 1,
    "name": "instagram-scraper-api",
    "title": "Instagram Scraper API",
    "description": "Scrape Instagram profiles, followers, following, posts, reels, and more. Runs as an HTTP API server in Standby mode.",
    "version": "2.0",
    "buildTag": "latest",
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

.actor/input_schema.json

{
    "title": "Instagram Scraper API",
    "type": "object",
    "schemaVersion": 1,
    "description": "Scrape Instagram profiles, followers, following, posts, and reels.",
    "properties": {
        "username": {
            "title": "Username",
            "type": "string",
            "description": "Instagram username to scrape.",
            "editor": "textfield"
        },
        "action": {
            "title": "Action",
            "type": "string",
            "description": "The scraping action to perform.",
            "enum": [
                "followers",
                "following",
                "userdetails",
                "latest-posts",
                "reels",
                "reel",
                "post"
            ],
            "editor": "select"
        },
        "url": {
            "title": "Reel URL",
            "type": "string",
            "description": "Instagram reel URL (only for the 'reel' action).",
            "editor": "textfield"
        },
        "shortcode": {
            "title": "Post Shortcode",
            "type": "string",
            "description": "Instagram post shortcode (only for the 'post' action).",
            "editor": "textfield"
        }
    },
    "required": ["action"]
}

config/config.go

package config

import (
	"encoding/json"
	"os"
	"strconv"
	"time"
)

type Config struct {
	IGAppID string

	// Connection pool
	MaxIdleConns        int
	MaxIdleConnsPerHost int
	MaxConnsPerHost     int
	IdleConnTimeout     time.Duration

	// Timeouts
	DialTimeout    time.Duration
	TLSTimeout     time.Duration
	RequestTimeout time.Duration

	// Concurrency
	ConcurrencyLimit int

	// Retry
	MaxRetries int
	BaseBackoff time.Duration
	MaxBackoff  time.Duration

	// Proxy
	ProxyList []string

	// Server
	PublicServerURL string
	UserInfoURL     string
}

func Load() *Config {
	return &Config{
		IGAppID: envOrDefault("IG_APP_ID", "936619743392459"),

		MaxIdleConns:        envIntOrDefault("IG_GLOBAL_MAX_CONNECTIONS", 800),
		MaxIdleConnsPerHost: envIntOrDefault("IG_PER_HOST_LIMIT", 80),
		MaxConnsPerHost:     envIntOrDefault("IG_PER_HOST_LIMIT", 80),
		IdleConnTimeout:     time.Duration(envIntOrDefault("IG_DNS_TTL_SECONDS", 300)) * time.Second,

		DialTimeout:    time.Duration(envFloatOrDefault("IG_SOCK_CONNECT_TIMEOUT", 4)) * time.Second,
		TLSTimeout:     time.Duration(envFloatOrDefault("IG_SOCK_CONNECT_TIMEOUT", 4)) * time.Second,
		RequestTimeout: time.Duration(envFloatOrDefault("IG_TOTAL_TIMEOUT", 15)) * time.Second,

		ConcurrencyLimit: envIntOrDefault("IG_CONCURRENCY_LIMIT", 300),

		MaxRetries:  3,
		BaseBackoff: time.Duration(envFloatOrDefault("IG_BASE_BACKOFF", 0.5)*1000) * time.Millisecond,
		MaxBackoff:  time.Duration(envFloatOrDefault("IG_MAX_BACKOFF", 8.0)) * time.Second,

		ProxyList: loadProxies(),

		PublicServerURL: os.Getenv("PUBLIC_SERVER_URL"),
		UserInfoURL:     envOrDefault("USERINFO_URL", "http://127.0.0.1/api/instagram/userinfo"),
	}
}

func envOrDefault(key, fallback string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return fallback
}

func envIntOrDefault(key string, fallback int) int {
	if v := os.Getenv(key); v != "" {
		if n, err := strconv.Atoi(v); err == nil {
			return n
		}
	}
	return fallback
}

func envFloatOrDefault(key string, fallback float64) float64 {
	if v := os.Getenv(key); v != "" {
		if f, err := strconv.ParseFloat(v, 64); err == nil {
			return f
		}
	}
	return fallback
}

// loadProxies reads proxy URLs from /app/config/proxies.json (baked into Docker image).
func loadProxies() []string {
	for _, path := range []string{"/app/config/proxies.json", "config/proxies.json"} {
		data, err := os.ReadFile(path)
		if err != nil {
			continue
		}
		var arr []string
		if err := json.Unmarshal(data, &arr); err != nil {
			continue
		}
		return arr
	}
	return nil
}

config/proxies.json

[
    "http://geonode_g0zRcloZra-type-residential:f3b6458a-1dbb-4ce2-8783-6d77911dccec@proxy.geonode.io:9000",
    "http://geonode_g0zRcloZra-type-residential:f3b6458a-1dbb-4ce2-8783-6d77911dccec@us.proxy.geonode.io:9000",
    "http://geonode_g0zRcloZra-type-residential:f3b6458a-1dbb-4ce2-8783-6d77911dccec@sg.proxy.geonode.io:9000"
]

instagram/client.go

package instagram

import (
	"compress/gzip"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"social-api/config"
	"social-api/proxy"
)

// Client is a high-throughput Instagram API client with connection pooling,
// proxy rotation, retry logic, and concurrency limiting.
type Client struct {
	cfg       *config.Config
	proxies   *proxy.Pool
	semaphore chan struct{} // concurrency limiter

	// proxyClient rotates through proxies
	proxyClient *http.Client
	// directClient bypasses proxies (fallback)
	directClient *http.Client
}

func NewClient(cfg *config.Config) *Client {
	pool := proxy.NewPool(cfg.ProxyList)

	baseTransport := &http.Transport{
		DialContext: (&net.Dialer{
			Timeout:   cfg.DialTimeout,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		TLSClientConfig:     &tls.Config{MinVersion: tls.VersionTLS12},
		TLSHandshakeTimeout: cfg.TLSTimeout,
		MaxIdleConns:        cfg.MaxIdleConns,
		MaxIdleConnsPerHost: cfg.MaxIdleConnsPerHost,
		MaxConnsPerHost:     cfg.MaxConnsPerHost,
		IdleConnTimeout:     cfg.IdleConnTimeout,
		ForceAttemptHTTP2:   true,
		DisableCompression:  false,
	}

	proxyTransport := pool.Transport(baseTransport)

	return &Client{
		cfg:       cfg,
		proxies:   pool,
		semaphore: make(chan struct{}, cfg.ConcurrencyLimit),
		proxyClient: &http.Client{
			Transport: proxyTransport,
			Timeout:   cfg.RequestTimeout,
		},
		directClient: &http.Client{
			Transport: baseTransport,
			Timeout:   cfg.RequestTimeout,
		},
	}
}

func (c *Client) Close() {
	c.proxyClient.CloseIdleConnections()
	c.directClient.CloseIdleConnections()
}

// RequestOption configures an HTTP request.
type RequestOption struct {
	Method   string
	URL      string
	Headers  map[string]string
	Cookies  map[string]string
	Params   map[string]string // query parameters
	FormData map[string]string // POST form data
	UseProxy bool
}

// DoWithRetry executes a request with retry logic and concurrency limiting.
// Returns the parsed JSON response body.
func (c *Client) DoWithRetry(ctx context.Context, opt RequestOption) (map[string]interface{}, error) {
	if opt.Method == "" {
		opt.Method = http.MethodGet
	}

	var lastErr error

	for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ {
		// Acquire semaphore
		select {
		case c.semaphore <- struct{}{}:
		case <-ctx.Done():
			return nil, ctx.Err()
		}

		result, err := c.doRequest(ctx, opt)
		<-c.semaphore // release

		if err == nil {
			return result, nil
		}

		lastErr = err
		log.Printf("attempt %d/%d failed: %v", attempt+1, c.cfg.MaxRetries+1, err)

		if attempt < c.cfg.MaxRetries {
			backoff := c.calcBackoff(attempt)
			select {
			case <-time.After(backoff):
			case <-ctx.Done():
				return nil, ctx.Err()
			}
		}
	}

	return nil, fmt.Errorf("all %d attempts failed: %w", c.cfg.MaxRetries+1, lastErr)
}

func (c *Client) doRequest(ctx context.Context, opt RequestOption) (map[string]interface{}, error) {
	client := c.directClient
	if opt.UseProxy {
		client = c.proxyClient
	}

	// Build URL with query params
	reqURL := opt.URL
	if len(opt.Params) > 0 && strings.ToUpper(opt.Method) == http.MethodGet {
		u, err := url.Parse(reqURL)
		if err != nil {
			return nil, err
		}
		q := u.Query()
		for k, v := range opt.Params {
			q.Set(k, v)
		}
		u.RawQuery = q.Encode()
		reqURL = u.String()
	}

	// Build body for POST
	var body io.Reader
	if strings.ToUpper(opt.Method) == http.MethodPost && len(opt.FormData) > 0 {
		form := url.Values{}
		for k, v := range opt.FormData {
			form.Set(k, v)
		}
		body = strings.NewReader(form.Encode())
	}

	req, err := http.NewRequestWithContext(ctx, strings.ToUpper(opt.Method), reqURL, body)
	if err != nil {
		return nil, err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	// Set headers
	for k, v := range opt.Headers {
		req.Header.Set(k, v)
	}

	// Set cookies
	for k, v := range opt.Cookies {
		req.AddCookie(&http.Cookie{Name: k, Value: v})
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("request error: %w", err)
	}
	defer resp.Body.Close()

	var reader io.Reader = resp.Body
	if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("gzip reader error: %w", err)
		}
		defer gr.Close()
		reader = gr
	}

	respBody, err := io.ReadAll(reader)
	if err != nil {
		return nil, fmt.Errorf("read body error: %w", err)
	}

	if resp.StatusCode == http.StatusOK {
		var result map[string]interface{}
		if err := json.Unmarshal(respBody, &result); err != nil {
			return nil, fmt.Errorf("json decode error: %w", err)
		}
		return result, nil
	}

	// Retryable status codes
	if resp.StatusCode == 429 || resp.StatusCode >= 500 {
		return nil, &RetryableError{Status: resp.StatusCode, Reason: resp.Status}
	}

	return map[string]interface{}{
		"error":       fmt.Sprintf("HTTP %d: %s", resp.StatusCode, resp.Status),
		"status_code": float64(resp.StatusCode),
	}, nil
}

func (c *Client) calcBackoff(attempt int) time.Duration {
	backoff := c.cfg.BaseBackoff * time.Duration(math.Pow(2, float64(attempt)))
	if backoff > c.cfg.MaxBackoff {
		backoff = c.cfg.MaxBackoff
	}
	jitter := time.Duration(rand.Float64()*300) * time.Millisecond
	return backoff + jitter
}

type RetryableError struct {
	Status int
	Reason string
}

func (e *RetryableError) Error() string {
	return fmt.Sprintf("retryable HTTP %d: %s", e.Status, e.Reason)
}

// DoRaw executes a request and returns the raw response body bytes and status.
// Used when callers need to do their own JSON parsing or handle non-JSON responses.
func (c *Client) DoRaw(ctx context.Context, opt RequestOption) ([]byte, int, error) {
	client := c.directClient
	if opt.UseProxy {
		client = c.proxyClient
	}

	reqURL := opt.URL
	if len(opt.Params) > 0 && strings.ToUpper(opt.Method) == http.MethodGet {
		u, err := url.Parse(reqURL)
		if err != nil {
			return nil, 0, err
		}
		q := u.Query()
		for k, v := range opt.Params {
			q.Set(k, v)
		}
		u.RawQuery = q.Encode()
		reqURL = u.String()
	}

	var body io.Reader
	if strings.ToUpper(opt.Method) == http.MethodPost && len(opt.FormData) > 0 {
		form := url.Values{}
		for k, v := range opt.FormData {
			form.Set(k, v)
		}
		body = strings.NewReader(form.Encode())
	}

	req, err := http.NewRequestWithContext(ctx, strings.ToUpper(opt.Method), reqURL, body)
	if err != nil {
		return nil, 0, err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	for k, v := range opt.Headers {
		req.Header.Set(k, v)
	}
	for k, v := range opt.Cookies {
		req.AddCookie(&http.Cookie{Name: k, Value: v})
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	var reader io.Reader = resp.Body
	if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
		gr, err := gzip.NewReader(resp.Body)
		if err != nil {
			return nil, resp.StatusCode, fmt.Errorf("gzip reader error: %w", err)
		}
		defer gr.Close()
		reader = gr
	}

	respBody, err := io.ReadAll(reader)
	if err != nil {
		return nil, resp.StatusCode, err
	}

	return respBody, resp.StatusCode, nil
}

instagram/encoding.go

package instagram

import (
	"fmt"
	"regexp"
	"strings"
)

const encodingChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"

// ShortcodeToPK converts an Instagram shortcode to its numeric media PK.
func ShortcodeToPK(shortcode string) string {
	base := uint64(len(encodingChars))
	var num uint64
	for _, ch := range shortcode {
		idx := strings.IndexRune(encodingChars, ch)
		if idx < 0 {
			continue
		}
		num = num*base + uint64(idx)
	}
	return fmt.Sprintf("%d", num)
}

var shortcodeRe = regexp.MustCompile(`/(?:reels?|p|tv)/([A-Za-z0-9_-]+)`)

// ExtractShortcode extracts the shortcode from an Instagram reel/post/tv URL.
func ExtractShortcode(rawURL string) string {
	matches := shortcodeRe.FindStringSubmatch(rawURL)
	if len(matches) >= 2 {
		return matches[1]
	}
	return ""
}

instagram/userinfo.go

package instagram

import (
	"context"
	"fmt"
	"math"
	"math/rand"
	"time"
)

type UserDetails struct {
	UserID         string
	FollowersCount int
	FollowingCount int
}

// ExtractUserDetails fetches user details directly from Instagram's web profile API.
func (c *Client) ExtractUserDetails(ctx context.Context, username string) (*UserDetails, error) {
	apiURL := fmt.Sprintf("https://i.instagram.com/api/v1/users/web_profile_info/?username=%s", username)

	result, err := c.DoWithRetry(ctx, RequestOption{
		Method:   "GET",
		URL:      apiURL,
		Headers:  map[string]string{"x-ig-app-id": c.cfg.IGAppID},
		UseProxy: true,
	})
	if err != nil {
		return nil, fmt.Errorf("failed to fetch user details: %w", err)
	}

	if errMsg, ok := result["error"]; ok {
		return nil, fmt.Errorf("instagram error: %v", errMsg)
	}

	// Navigate: data -> user
	data, _ := result["data"].(map[string]interface{})
	user, _ := data["user"].(map[string]interface{})
	if user == nil {
		return nil, fmt.Errorf("user not found in response")
	}

	userID := stringFromMap(user, "id")
	if userID == "" {
		return nil, fmt.Errorf("failed to extract user ID from response")
	}

	followedBy, _ := user["edge_followed_by"].(map[string]interface{})
	follow, _ := user["edge_follow"].(map[string]interface{})

	return &UserDetails{
		UserID:         userID,
		FollowersCount: intFromMap(followedBy, "count"),
		FollowingCount: intFromMap(follow, "count"),
	}, nil
}

func (c *Client) backoffSleep(ctx context.Context, attempt int) {
	backoff := c.cfg.BaseBackoff * time.Duration(math.Pow(2, float64(attempt)))
	if backoff > c.cfg.MaxBackoff {
		backoff = c.cfg.MaxBackoff
	}
	jitter := time.Duration(rand.Float64()*300) * time.Millisecond
	select {
	case <-time.After(backoff + jitter):
	case <-ctx.Done():
	}
}

func stringFromMap(m map[string]interface{}, key string) string {
	v, ok := m[key]
	if !ok || v == nil {
		return ""
	}
	switch val := v.(type) {
	case string:
		return val
	case float64:
		return fmt.Sprintf("%.0f", val)
	default:
		return fmt.Sprintf("%v", val)
	}
}

func intFromMap(m map[string]interface{}, key string) int {
	v, ok := m[key]
	if !ok || v == nil {
		return 0
	}
	switch val := v.(type) {
	case float64:
		return int(val)
	case int:
		return val
	default:
		return 0
	}
}

middleware/middleware.go

package middleware

import (
	"net/http"
)

// CORS adds permissive CORS headers (matching the Python allow_origins=["*"]).
func CORS(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Methods", "*")
		w.Header().Set("Access-Control-Allow-Headers", "*")
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		if r.Method == http.MethodOptions {
			w.WriteHeader(http.StatusNoContent)
			return
		}
		next.ServeHTTP(w, r)
	})
}

proxy/proxy.go

package proxy

import (
	"math/rand"
	"net/http"
	"net/url"
	"sync"
)

// Pool manages a list of proxy URLs and provides random selection.
type Pool struct {
	mu      sync.RWMutex
	proxies []*url.URL
	raw     []string
}

func NewPool(rawURLs []string) *Pool {
	p := &Pool{raw: rawURLs}
	for _, raw := range rawURLs {
		if u, err := url.Parse(raw); err == nil {
			p.proxies = append(p.proxies, u)
		}
	}
	return p
}

// Random returns a random proxy URL or nil if pool is empty.
func (p *Pool) Random() *url.URL {
	p.mu.RLock()
	defer p.mu.RUnlock()
	if len(p.proxies) == 0 {
		return nil
	}
	return p.proxies[rand.Intn(len(p.proxies))]
}

// Transport returns an *http.Transport configured to use a random proxy.
func (p *Pool) Transport(base *http.Transport) *http.Transport {
	t := base.Clone()
	t.Proxy = func(req *http.Request) (*url.URL, error) {
		return p.Random(), nil
	}
	return t
}

server/handlers.go

package server

import (
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"social-api/instagram"
)

// sessionCookies extracts session cookies from the session_id query parameter.
func sessionCookies(r *http.Request) map[string]string {
	cookies := make(map[string]string)
	if sessionID := r.URL.Query().Get("session_id"); sessionID != "" {
		cookies["sessionid"] = sessionID
		if parts := strings.SplitN(sessionID, ":", 2); len(parts) >= 1 && parts[0] != "" {
			cookies["ds_user_id"] = parts[0]
		}
	}
	return cookies
}

// ───── Followers ─────

func (s *Server) handleFollowers(w http.ResponseWriter, r *http.Request) {
	username := r.URL.Query().Get("username")
	if username == "" {
		errorJSON(w, http.StatusBadRequest, "username is required")
		return
	}

	ctx := r.Context()
	user, err := s.client.ExtractUserDetails(ctx, username)
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	if user.UserID == "" {
		writeJSON(w, http.StatusNotFound, map[string]interface{}{"details": "User not found.", "status": 404})
		return
	}

	apiURL := fmt.Sprintf("https://www.instagram.com/api/v1/friendships/%s/followers/", user.UserID)
	totalAvailable := user.FollowersCount
	target := totalAvailable
	if target > 500 {
		target = 500
	}
	if target <= 0 {
		writeJSON(w, http.StatusOK, map[string]interface{}{"followers": []interface{}{}})
		return
	}

	cookies := sessionCookies(r)

	collected := make([]interface{}, 0, target)
	seen := make(map[string]bool)
	var nextMaxID string

	for len(collected) < target {
		params := map[string]string{
			"count":          "12",
			"search_surface": "follow_list_page",
		}
		if nextMaxID != "" {
			params["max_id"] = nextMaxID
		}

		result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
			Method:   "GET",
			URL:      apiURL,
			Headers:  map[string]string{"x-ig-app-id": s.cfg.IGAppID},
			Params:   params,
			Cookies:  cookies,
			UseProxy: true,
		})
		if err != nil {
			break
		}
		if errMsg, ok := result["error"]; ok {
			fmt.Printf("Failed to fetch followers: %v\n", errMsg)
			break
		}

		users, _ := result["users"].([]interface{})
		for _, u := range users {
			um, ok := u.(map[string]interface{})
			if !ok {
				continue
			}
			pkID := fmt.Sprintf("%v", um["pk_id"])
			if pkID != "" && pkID != "<nil>" && !seen[pkID] {
				seen[pkID] = true
				collected = append(collected, um)
				if len(collected) >= target {
					break
				}
			}
		}

		hasMore, _ := result["has_more"].(bool)
		nm, _ := result["next_max_id"]
		nextMaxID = fmt.Sprintf("%v", nm)
		restricted := result["should_limit_list_of_followers"] != nil || result["use_clickable_see_more"] != nil

		if !hasMore || nextMaxID == "" || nextMaxID == "<nil>" || restricted {
			break
		}
	}

	writeJSON(w, http.StatusOK, map[string]interface{}{"followers": collected})
}

// ───── Following ─────

func (s *Server) handleFollowing(w http.ResponseWriter, r *http.Request) {
	username := r.URL.Query().Get("username")
	if username == "" {
		errorJSON(w, http.StatusBadRequest, "username is required")
		return
	}

	ctx := r.Context()
	user, err := s.client.ExtractUserDetails(ctx, username)
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	if user.UserID == "" {
		writeJSON(w, http.StatusNotFound, map[string]interface{}{"details": "User not found.", "status": 404})
		return
	}

	apiURL := fmt.Sprintf("https://www.instagram.com/api/v1/friendships/%s/following/", user.UserID)
	totalAvailable := user.FollowingCount
	target := totalAvailable
	if target > 500 {
		target = 500
	}
	if target <= 0 {
		writeJSON(w, http.StatusOK, map[string]interface{}{"following": []interface{}{}})
		return
	}

	cookies := sessionCookies(r)

	collected := make([]interface{}, 0, target)
	seen := make(map[string]bool)
	var nextMaxID string

	for len(collected) < target {
		params := map[string]string{"count": "12"}
		if nextMaxID != "" {
			params["max_id"] = nextMaxID
		}

		result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
			Method:   "GET",
			URL:      apiURL,
			Headers:  map[string]string{"x-ig-app-id": s.cfg.IGAppID},
			Params:   params,
			Cookies:  cookies,
			UseProxy: true,
		})
		if err != nil {
			break
		}
		if errMsg, ok := result["error"]; ok {
			fmt.Printf("Failed to fetch following: %v\n", errMsg)
			break
		}

		users, _ := result["users"].([]interface{})
		for _, u := range users {
			um, ok := u.(map[string]interface{})
			if !ok {
				continue
			}
			pkID := fmt.Sprintf("%v", um["pk_id"])
			if pkID != "" && pkID != "<nil>" && !seen[pkID] {
				seen[pkID] = true
				collected = append(collected, um)
				if len(collected) >= target {
					break
				}
			}
		}

		hasMore, _ := result["has_more"].(bool)
		nm, _ := result["next_max_id"]
		nextMaxID = fmt.Sprintf("%v", nm)
		restricted := result["should_limit_list_of_followers"] != nil || result["use_clickable_see_more"] != nil

		if !hasMore || nextMaxID == "" || nextMaxID == "<nil>" || restricted {
			break
		}
	}

	writeJSON(w, http.StatusOK, map[string]interface{}{"following": collected})
}

// ───── User Details ─────

func (s *Server) handleUserDetails(w http.ResponseWriter, r *http.Request) {


	username := r.URL.Query().Get("username")
	if username == "" {
		errorJSON(w, http.StatusBadRequest, "username is required")
		return
	}

	ctx := r.Context()
	result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
		Method:   "GET",
		URL:      fmt.Sprintf("https://i.instagram.com/api/v1/users/web_profile_info/?username=%s", username),
		Headers:  map[string]string{"x-ig-app-id": s.cfg.IGAppID},
		UseProxy: true,
	})
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	if errMsg, ok := result["error"]; ok {
		status := http.StatusInternalServerError
		if sc, ok := result["status_code"].(float64); ok {
			status = int(sc)
		}
		errorJSON(w, status, fmt.Sprintf("Failed to fetch user details: %v", errMsg))
		return
	}

	writeJSON(w, http.StatusOK, result)
}

// ───── Latest Posts ─────

func (s *Server) handleLatestPosts(w http.ResponseWriter, r *http.Request) {


	username := r.URL.Query().Get("username")
	if username == "" {
		errorJSON(w, http.StatusBadRequest, "username is required")
		return
	}

	ctx := r.Context()
	result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
		Method:   "GET",
		URL:      fmt.Sprintf("https://i.instagram.com/api/v1/users/web_profile_info/?username=%s", username),
		Headers:  map[string]string{"x-ig-app-id": s.cfg.IGAppID},
		UseProxy: true,
	})
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	if errMsg, ok := result["error"]; ok {
		errorJSON(w, http.StatusInternalServerError, fmt.Sprintf("Failed to fetch user details: %v", errMsg))
		return
	}

	// Navigate: data -> user -> edge_owner_to_timeline_media -> edges
	data, _ := result["data"].(map[string]interface{})
	user, _ := data["user"].(map[string]interface{})
	timeline, _ := user["edge_owner_to_timeline_media"].(map[string]interface{})
	edges, _ := timeline["edges"].([]interface{})

	// Filter out pinned posts
	filtered := make([]interface{}, 0, len(edges))
	for _, e := range edges {
		edge, ok := e.(map[string]interface{})
		if !ok {
			continue
		}
		if isPinned(edge) {
			continue
		}
		filtered = append(filtered, edge)
	}

	writeJSON(w, http.StatusOK, filtered)
}

func isPinned(edge map[string]interface{}) bool {
	node, _ := edge["node"].(map[string]interface{})
	if node == nil {
		return false
	}
	if pinned, ok := node["pinned_for_users"].([]interface{}); ok && len(pinned) > 0 {
		return true
	}
	if pinned, ok := node["timeline_pinned_user_ids"].([]interface{}); ok && len(pinned) > 0 {
		return true
	}
	return false
}

// ───── Reels (all) ─────

func (s *Server) handleReels(w http.ResponseWriter, r *http.Request) {


	username := r.URL.Query().Get("username")
	if username == "" {
		errorJSON(w, http.StatusBadRequest, "username is required")
		return
	}

	ctx := r.Context()
	user, err := s.client.ExtractUserDetails(ctx, username)
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}
	if user.UserID == "" {
		writeJSON(w, http.StatusNotFound, map[string]interface{}{"details": "User not found.", "status": 404})
		return
	}

	cookies := sessionCookies(r)

	// Scrape CSRF token like the Python version does
	csrfToken := ""
	body, _, scrapeErr := s.client.DoRaw(ctx, instagram.RequestOption{
		Method:   "GET",
		URL:      "https://www.instagram.com/",
		Headers:  postHeaders(),
		UseProxy: true,
	})
	if scrapeErr == nil && body != nil {
		csrfToken = extractCSRFFromHTML(string(body))
	}

	reelsHeaders := map[string]string{"x-ig-app-id": s.cfg.IGAppID}
	if csrfToken != "" {
		reelsHeaders["X-CSRFToken"] = csrfToken
	}

	var allEdges []interface{}
	var endCursor string
	hasNextPage := true
	pagesFetched := 0

	for page := 0; page < 10; page++ {
		var formData map[string]string

		if page == 0 {
			formData = map[string]string{
				"variables": fmt.Sprintf(`{"data":{"include_feed_video":true,"page_size":12,"target_user_id":"%s"}}`, user.UserID),
				"doc_id":    "24127588873492897",
			}
		} else {
			if !hasNextPage || endCursor == "" {
				break
			}
			formData = map[string]string{
				"variables": fmt.Sprintf(`{"after":"%s","before":null,"data":{"include_feed_video":true,"page_size":12,"target_user_id":"%s"},"first":3,"last":null}`, endCursor, user.UserID),
				"doc_id":    "24127588873492897",
			}
		}

		result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
			Method:   "POST",
			URL:      "https://www.instagram.com/graphql/query",
			Headers:  reelsHeaders,
			FormData: formData,
			Cookies:  cookies,
			UseProxy: true,
		})
		if err != nil {
			errorJSON(w, http.StatusInternalServerError, fmt.Sprintf("Failed on page %d: %v", page+1, err))
			return
		}
		if errMsg, ok := result["error"]; ok {
			errorJSON(w, http.StatusInternalServerError, fmt.Sprintf("Failed on page %d: %v", page+1, errMsg))
			return
		}

		data, _ := result["data"].(map[string]interface{})
		connection, _ := data["xdt_api__v1__clips__user__connection_v2"].(map[string]interface{})
		edges, _ := connection["edges"].([]interface{})
		pageInfo, _ := connection["page_info"].(map[string]interface{})

		allEdges = append(allEdges, edges...)
		pagesFetched = page + 1

		endCursor, _ = pageInfo["end_cursor"].(string)
		hasNextPage, _ = pageInfo["has_next_page"].(bool)

		if !hasNextPage {
			break
		}
	}

	writeJSON(w, http.StatusOK, map[string]interface{}{
		"username":      username,
		"user_id":       user.UserID,
		"pages_fetched": pagesFetched,
		"total_reels":   len(allEdges),
		"edges":         allEdges,
	})
}

// ───── Single Reel Info ─────

func (s *Server) handleReelInfo(w http.ResponseWriter, r *http.Request) {


	rawURL := r.URL.Query().Get("url")
	if rawURL == "" {
		errorJSON(w, http.StatusBadRequest, "url is required")
		return
	}

	shortcode := instagram.ExtractShortcode(rawURL)
	if shortcode == "" {
		errorJSON(w, http.StatusBadRequest, "Could not extract shortcode from URL")
		return
	}

	mediaPK := instagram.ShortcodeToPK(shortcode)
	apiURL := fmt.Sprintf("https://i.instagram.com/api/v1/media/%s/info/", mediaPK)

	ctx := r.Context()
	reelHeaders := map[string]string{
		"User-Agent":              "Instagram 364.0.0.35.86 Android (26/8.0.0; 480dpi; 1080x1920; OnePlus; 6T Dev; devitron; qcom; en_US; 374010953)",
		"Accept-Language":         "en-US",
		"Accept-Encoding":        "gzip, deflate",
		"X-IG-App-ID":            "567067343352427",
		"X-IG-Capabilities":      "3brTv10=",
		"X-IG-Connection-Type":   "WIFI",
		"X-IG-Timezone-Offset":   "0",
		"X-Pigeon-Rawclienttime": fmt.Sprintf("%.3f", float64(time.Now().UnixMilli())/1000),
		"X-IG-Bandwidth-Speed-KBPS": strconv.Itoa(2500 + rand.Intn(500)),
		"Host":                   "i.instagram.com",
		"Connection":             "keep-alive",
	}

	cookies := sessionCookies(r)

	result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
		Method:   "GET",
		URL:      apiURL,
		Headers:  reelHeaders,
		Cookies:  cookies,
		UseProxy: true,
	})
	if err != nil {
		errorJSON(w, http.StatusServiceUnavailable, fmt.Sprintf("Failed after retries: %v", err))
		return
	}
	if errMsg, ok := result["error"]; ok {
		status := http.StatusInternalServerError
		if sc, ok := result["status_code"].(float64); ok {
			status = int(sc)
		}
		errorJSON(w, status, fmt.Sprintf("%v", errMsg))
		return
	}

	items, _ := result["items"].([]interface{})
	if len(items) == 0 {
		errorJSON(w, http.StatusNotFound, "No items returned")
		return
	}

	writeJSON(w, http.StatusOK, items[0])
}

// ───── Post by Shortcode ─────

func (s *Server) handlePost(w http.ResponseWriter, r *http.Request) {


	shortcode := r.URL.Query().Get("shortcode")
	if shortcode == "" {
		errorJSON(w, http.StatusBadRequest, "shortcode is required")
		return
	}

	ctx := r.Context()

	// Step 1: Visit instagram.com to get CSRF token (like the Python version)
	csrfToken := ""
	body, _, err := s.client.DoRaw(ctx, instagram.RequestOption{
		Method:   "GET",
		URL:      "https://www.instagram.com/",
		Headers:  postHeaders(),
		UseProxy: true,
	})
	if err == nil && body != nil {
		// Try to extract csrftoken from Set-Cookie-like patterns in response
		// For simplicity, use a basic approach - the Go http.Client handles cookies
		// but we need to extract from raw response
		csrfToken = extractCSRFFromHTML(string(body))
	}

	// Step 2: Make the GraphQL query
	formData := map[string]string{
		"variables":         fmt.Sprintf(`{"shortcode":"%s"}`, shortcode),
		"doc_id":            "8845758582119845",
		"server_timestamps": "true",
	}

	hdrs := postHeaders()
	if csrfToken != "" {
		hdrs["X-CSRFToken"] = csrfToken
	}

	result, err := s.client.DoWithRetry(ctx, instagram.RequestOption{
		Method:   "POST",
		URL:      "https://www.instagram.com/graphql/query",
		Headers:  hdrs,
		FormData: formData,
		UseProxy: true,
	})
	if err != nil {
		errorJSON(w, http.StatusInternalServerError, err.Error())
		return
	}

	writeJSON(w, http.StatusOK, result)
}

func postHeaders() map[string]string {
	return map[string]string{
		"Accept-Encoding": "gzip, deflate",
		"Accept-Language": "en-US,en;q=0.8",
		"Referer":         "https://www.instagram.com/",
		"User-Agent":      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
		"Accept":          "*/*",
	}
}

func extractCSRFFromHTML(html string) string {
	// Look for csrftoken in various patterns
	patterns := []string{
		`"csrf_token":"`,
		`"csrfToken":"`,
		`csrftoken=`,
	}
	for _, p := range patterns {
		idx := indexOf(html, p)
		if idx < 0 {
			continue
		}
		start := idx + len(p)
		end := start
		for end < len(html) && html[end] != '"' && html[end] != ';' && html[end] != '&' {
			end++
		}
		if end > start {
			return html[start:end]
		}
	}
	return ""
}

func indexOf(s, substr string) int {
	for i := 0; i <= len(s)-len(substr); i++ {
		if s[i:i+len(substr)] == substr {
			return i
		}
	}
	return -1
}

server/json.go

package server

import (
	"encoding/json"
	"net/http"
)

func writeJSON(w http.ResponseWriter, status int, v interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	json.NewEncoder(w).Encode(v)
}

func errorJSON(w http.ResponseWriter, status int, msg string) {
	writeJSON(w, status, map[string]interface{}{
		"error":       msg,
		"status_code": status,
	})
}

server/server.go

package server

import (
	"context"
	"net/http"
	"time"

	"social-api/config"
	"social-api/instagram"
	"social-api/middleware"
)

type Server struct {
	cfg    *config.Config
	client *instagram.Client
	http   *http.Server
	mux    *http.ServeMux
}

func New() *Server {
	cfg := config.Load()
	client := instagram.NewClient(cfg)

	mux := http.NewServeMux()
	s := &Server{
		cfg:    cfg,
		client: client,
		mux:    mux,
	}

	s.registerRoutes()
	return s
}

func (s *Server) registerRoutes() {
	// Health check for Apify Standby readiness
	s.mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write([]byte(`{"status":"ok"}`))
	})

	s.mux.HandleFunc("GET /api/instagram/followers", s.handleFollowers)
	s.mux.HandleFunc("GET /api/instargam/following", s.handleFollowing) // preserve typo from original
	s.mux.HandleFunc("GET /api/instagram/following", s.handleFollowing) // also register correct spelling
	s.mux.HandleFunc("GET /api/instagram/userdetails", s.handleUserDetails)
	s.mux.HandleFunc("GET /api/instagram/latest-posts", s.handleLatestPosts)
	s.mux.HandleFunc("GET /api/instagram/reels", s.handleReels)
	s.mux.HandleFunc("GET /api/instagram/reel", s.handleReelInfo)
	s.mux.HandleFunc("GET /api/instagram/post", s.handlePost)
}

func (s *Server) Start(addr string) error {
	s.http = &http.Server{
		Addr:              addr,
		Handler:           middleware.CORS(s.mux),
		ReadHeaderTimeout: 10 * time.Second,
		ReadTimeout:       30 * time.Second,
		WriteTimeout:      120 * time.Second, // long for paginated endpoints
		IdleTimeout:       120 * time.Second,
		MaxHeaderBytes:    1 << 20,
	}
	return s.http.ListenAndServe()
}

func (s *Server) Shutdown() {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	s.http.Shutdown(ctx)
	s.client.Close()
}

Instagram LIST / REMOVE Followings

saswave/instagram-list-remove-followings

List all the accounts you are following for data analysis. Then provide a list of following to be removed. Help you clean your instagram feed page by removing bot accounts and not active followers

SASWAVE

Sitemap Change Detector

tri_angle/sitemap-change-detector

Identify and monitor sitemaps for specified websites. Retrieve only the new, updated, or removed URLs since the last crawl.

Tri⟁angle

Company Jobs Tracker — Multi-Portal Hiring Monitor

blackfalcondata/company-jobs-tracker

Track job listings for specific companies across StepStone portals (DE, AT, BE, NL, Totaljobs, NIjobs, Pnet). Detects new, changed, and removed postings with stateful diff tracking. Pay per company tracked.

Black Falcon Data

Instagram LIST / REMOVE Followers

saswave/instagram-list-remove-followers

List all your followers for data analysis. Then provide a list of followers to be removed. Clean your account bots followers at scale. Help you to boost reach and engagements on your posts by removing bot accounts and not active followers

SASWAVE

198

5.0

⚖️ Website Terms & Policy Monitor

taroyamada/saas-change-monitor-actor-premium

Extract and monitor website terms of service, privacy policies, and legal pages to detect added clauses or removed text via SHA256 diffs.

太郎山田

Daraz Scraper

stylish_flea/daraz-scraper

Daraz Scraper gathers public product data for research, fully respects Daraz’s Terms, and will be removed upon Daraz’s request.

MY Data

3.0

Shopify Competitor Monitor - New Products & Price Changes

equitable_xylulose/shopify-competitor-monitor

Monitor Shopify competitor stores for new products, price changes, and removed products. Schedule daily runs to never miss a competitor move.

Kong Qinglu

Apify Smart Dataset Comparator

agenscrape/apify-smart-dataset-comparator

Compare 2-10 Apify datasets to detect changes, new/removed records, and duplicates. Features field-level diffs, smart merging, schema validation, data cleaning, and anomaly detection. Perfect for price monitoring, lead deduplication, and data quality tracking.

Agenscrape

Sitemap Diff Tool

automation-lab/sitemap-diff-tool

Compare two XML sitemaps and find added, removed, or changed URLs. Detects lastmod, priority, and changefreq changes. Supports sitemap index files. Export results as JSON, CSV, or Excel.

Stas Persiianenko

Website Changes Detector

tri_angle/website-changes-detector

Automatically track website changes (new, updated, or removed pages) using periodic re-crawling. Supports flexible crawling settings, a list of URLs or sections of websites, antiblocking, filtering by keywords, and more.

Tri⟁angle

5.0

depracted (to be removed)

Dockerfile

go.mod

go.sum

main.go

.actor/Dockerfile

.actor/actor.json

.actor/input_schema.json

config/config.go

config/proxies.json

instagram/client.go

instagram/encoding.go

instagram/userinfo.go

middleware/middleware.go

proxy/proxy.go

server/handlers.go

server/json.go

server/server.go

You might also like

Instagram LIST / REMOVE Followings

Sitemap Change Detector

Company Jobs Tracker — Multi-Portal Hiring Monitor

Instagram LIST / REMOVE Followers

⚖️ Website Terms & Policy Monitor

Daraz Scraper

Shopify Competitor Monitor - New Products & Price Changes

Apify Smart Dataset Comparator

Sitemap Diff Tool

Website Changes Detector

Dockerfile

go.mod

go.sum

main.go

.actor/Dockerfile

.actor/actor.json

.actor/input_schema.json

config/config.go

config/proxies.json

instagram/client.go

instagram/encoding.go

instagram/userinfo.go

middleware/middleware.go

proxy/proxy.go

server/handlers.go

server/json.go

server/server.go