Actor picture

Big Query

juansgaitan/big-query

Append a CSV file to a Google bigQuery table. Create a "Service credentials" at the https://console.cloud.google.com/, copy & paste the JSON file into the variable value for 'CREDENTIALS' and set it as a secret.

No credit card required

Author's avatarJuan Gaitán Villamizar
  • Modified
  • Users5
  • Runs154

Based on the apify/actor-node-basic:v0.21.10 Docker image (see docs).

This file is 224 lines long. Only the first 50 are shown. Show all

// Source file for Hosted source in 'Source type'
const fs = require('fs');
const Apify = require('apify');
const json2csv = require('json2csv'); // eslint-disable-line
const BigQuery = require('@google-cloud/bigquery'); // eslint-disable-line

const { log, error } = console;

let isStoreIdSet = false;
async function storeOrGetResults(key, items = [], filterKey) {
  if (!isStoreIdSet || !key) {
    throw new Error(`Error while storing or getting results. Missing ${key ?
      'storeId in store' : 'key value'}.`);
  }

  const { keyValueStores } = Apify.client;
  const record = await keyValueStores.getRecord({ key });
  const storeRecord = record && record.body ? record.body : [];
  let previous = typeof storeRecord === 'string' ? JSON.parse(storeRecord) : storeRecord;

  if (items.length === 0) {
    return { previous };
  }

  const current = items.slice();
  if (current.length && previous.length && filterKey) {
    const cache = current.reduce((object, item) => (
      Object.assign(object, { [item[filterKey]]: true })
    ), {});
    previous = previous.filter(item => !cache[item[filterKey]]);
  }

  const next = previous.concat(current);
  if (previous.length !== current.length) {
    await keyValueStores.putRecord({
      key,
      body: JSON.stringify(next)
    });
  } else {
    log('No state modifications required.');
  }

  log('Previous results:', previous.length);
  log('Current results:', current.length);
  log('Next results:', next.length);
  return { previous, current, next };
}

async function createDataset(datasetName, bigquery) {
  const [datasets] = await bigquery.getDatasets();