大量データをcsvで投入するためのスクリプトを作ってみた(TypeScript)

やりたいこと

csvからDBへテストデータを投入するため、大量データをcsvで用意したい。 fsのcreateWriteStreamを使ってcsvを出力する処理を作ってみた。

◇ディレクトリ構成

root/
  ├ csv/
  │  ├ sample/
  │  └ seedCsv.ts
  └ package.json

◇ソースコード

package.json

{
  "name": "test_node",
  "version": "1.0.0",
  "main": "index.js",
  "license": "MIT",
  "devDependencies": {
    "@types/node": "^17.0.23",
    "ts-node": "^10.7.0",
    "typescript": "^4.6.3"
  }
}

csv/seedCsv.ts

import * as fs from "fs";
import * as path from "path";

function getRandomInt(min: number, max: number) {
  min = Math.ceil(min);
  max = Math.floor(max);
  return Math.floor(Math.random() * (max - min) + min);
}

const prefix = "SamplePrefix";
const startTimestamp = 1648134000000;
const diff = 60000;
const count = 10;

const types = [
  "sample1",
  "sample2",
  "sample3"
];

for (const type of types) {
  const DATA_PATH = path.join(__dirname, `./${type}.csv`);
  const dataWriteStream = fs.createWriteStream(DATA_PATH);

  const header = "id,meta,timestamp,type,value,ttlDay";
  dataWriteStream.write(header + "\n");

  for (let i = 0; i < count; i++) {
    const timestamp = startTimestamp + diff * i;
    const value = getRandomInt(0, 100);
    let record = "";
    record += `${prefix}:${type}:0000SAMPLE${i}`;
    record += `,${prefix}`;
    record += `,${timestamp}`;
    record += `,${type}`;
    record += `,${value}`;
    record += `,${5}`;
    dataWriteStream.write(record + "\n");
  }
}

簡単に解説

function getRandomInt(min: number, max: number) {
  min = Math.ceil(min);
  max = Math.floor(max);
  return Math.floor(Math.random() * (max - min) + min);

getRandomInt()によってmin, maxを指定しランダムな整数を得る

const dataWriteStream = fs.createWriteStream(DATA_PATH);

でwriteStreamのインスタンスを生成

dataWriteStream.write(header + "\n")

ここでcsvに一行ずつ書き込みを行う

const diff = 60000;

timestampを1分ごとにインクリメントしたいため、diffを60000ミリ秒で設定しました

実行コマンド

$ cd root/
$ yarn install
$ yarn ts-node csv/seedCsv.ts

結果

sample以下にcsvファイルが生成されました

sample/
   ├sample1.csv
   ├sample2.csv
   └sample3.csv

sample1.csv

id,meta,timestamp,type,value,ttlDay
SamplePrefix:sample1:0000SAMPLE0,SamplePrefix,1648134000000,sample1,24,5
SamplePrefix:sample1:0000SAMPLE1,SamplePrefix,1648134060000,sample1,66,5
SamplePrefix:sample1:0000SAMPLE2,SamplePrefix,1648134120000,sample1,74,5
SamplePrefix:sample1:0000SAMPLE3,SamplePrefix,1648134180000,sample1,51,5
SamplePrefix:sample1:0000SAMPLE4,SamplePrefix,1648134240000,sample1,52,5
SamplePrefix:sample1:0000SAMPLE5,SamplePrefix,1648134300000,sample1,48,5
SamplePrefix:sample1:0000SAMPLE6,SamplePrefix,1648134360000,sample1,91,5
SamplePrefix:sample1:0000SAMPLE7,SamplePrefix,1648134420000,sample1,12,5
SamplePrefix:sample1:0000SAMPLE8,SamplePrefix,1648134480000,sample1,70,5
SamplePrefix:sample1:0000SAMPLE9,SamplePrefix,1648134540000,sample1,92,5

sample2.csv

id,meta,timestamp,type,value,ttlDay
SamplePrefix:sample2:0000SAMPLE0,SamplePrefix,1648134000000,sample2,42,5
SamplePrefix:sample2:0000SAMPLE1,SamplePrefix,1648134060000,sample2,81,5
SamplePrefix:sample2:0000SAMPLE2,SamplePrefix,1648134120000,sample2,6,5
SamplePrefix:sample2:0000SAMPLE3,SamplePrefix,1648134180000,sample2,55,5
SamplePrefix:sample2:0000SAMPLE4,SamplePrefix,1648134240000,sample2,32,5
SamplePrefix:sample2:0000SAMPLE5,SamplePrefix,1648134300000,sample2,28,5
SamplePrefix:sample2:0000SAMPLE6,SamplePrefix,1648134360000,sample2,74,5
SamplePrefix:sample2:0000SAMPLE7,SamplePrefix,1648134420000,sample2,96,5
SamplePrefix:sample2:0000SAMPLE8,SamplePrefix,1648134480000,sample2,58,5
SamplePrefix:sample2:0000SAMPLE9,SamplePrefix,1648134540000,sample2,16,5

sample3.csv

id,meta,timestamp,type,value,ttlDay
SamplePrefix:sample3:0000SAMPLE0,SamplePrefix,1648134000000,sample3,64,5
SamplePrefix:sample3:0000SAMPLE1,SamplePrefix,1648134060000,sample3,49,5
SamplePrefix:sample3:0000SAMPLE2,SamplePrefix,1648134120000,sample3,10,5
SamplePrefix:sample3:0000SAMPLE3,SamplePrefix,1648134180000,sample3,22,5
SamplePrefix:sample3:0000SAMPLE4,SamplePrefix,1648134240000,sample3,56,5
SamplePrefix:sample3:0000SAMPLE5,SamplePrefix,1648134300000,sample3,90,5
SamplePrefix:sample3:0000SAMPLE6,SamplePrefix,1648134360000,sample3,21,5
SamplePrefix:sample3:0000SAMPLE7,SamplePrefix,1648134420000,sample3,91,5
SamplePrefix:sample3:0000SAMPLE8,SamplePrefix,1648134480000,sample3,59,5
SamplePrefix:sample3:0000SAMPLE9,SamplePrefix,1648134540000,sample3,94,5