tf.util.shuffle(data)

      const epsilon = tf.scalar(1e-4) // 创建一个很小的标量epsilon
      const inputs = data.map((d) => d.horsepower)
      const labels = data.map((d) => d.mpg)

      const inputTensor = tf.tensor2d(inputs, [inputs.length, 1])
      const labelTensor = tf.tensor2d(labels, [labels.length, 1])

      const sqrt_pow_mean = inputTensor
        .square()
        .mean(1)
        .sqrt()
        .expandDims(1)
        .add(epsilon)
      const label_pow_mean = inputTensor
        .square()
        .mean(1)
        .sqrt()
        .expandDims(1)
        .add(epsilon)
        

 //import * as assert from 'assert'
import { getEncoding, encodingForModel } from 'js-tiktoken'
import * as fs from 'fs'
import * as nj from 'numjs'
//import * as os from 'os'
import * as path from 'path'
//import * as tf from '@tensorflow/tfjs'
//import { fileURLToPath } from 'url'

//const __filename = fileURLToPath(import.meta.url)
//const __dirname = path.dirname(__filename)
const enc = getEncoding('gpt2')
//assert.strictEqual(enc.decode(enc.encode('hello world')), 'hello world')

fs.readFile('./night.txt', 'utf8', (err: Error | null, content: string) => {
  if (err) {
    console.error('Error reading file:', err)
    return
  }
  console.log(content.length)
  const n = content.length
  const train_data = content.slice(0, Math.floor(n * 0.8)) //切片，前80%
  const val_data = content.slice(Math.floor(n * 0.8)) //后20%
  let train_ids = enc.encode(train_data) //编码
  let val_ids = enc.encode(val_data)
  console.log('train tokens:', train_ids.length)
  console.log('val tokens:', val_ids.length)

  var train_ids_list = nj.array(train_ids, 'uint16')
  var val_ids_list = nj.array(val_ids, 'uint16')

  const trainArray = train_ids_list.tolist() // 转换为普通数组
  const trainBuffer = Buffer.from(trainArray) // 转换为 Buffer 对象
  const valArray = train_ids_list.tolist()
  const valBuffer = Buffer.from(valArray)

  fs.writeFileSync(path.join('./data', 'train.bin'), trainBuffer)

  fs.writeFileSync(path.join('./data', 'val.bin'), valBuffer)
})