node.js : a file system that replicates to S3

Sometimes it would be handy to use S3 like you used a local filesystem.
fs.writeFile would write a file, fs.readFile would read one, and fs.stat would use get some information about it.
It would be a bucket that you can read from a local cache if you need to read the same file twice.

Nothing can prevent you from implementing a drop-in module for `fs` that has the ability to replicate data from/to S3.
Below is a simple set of functions that you can use to re-implement with S3 synchronization :

import { promises as fs } from “fs”;
import {
  PathLike,
  OpenMode,
  Mode,
  ObjectEncodingOptions,
  StatOptions,
  Stats,
from “node:fs”;
import { RequestSigner } from “aws4”;
import { resolve } from “path”;
import { fetch } from “./cheap-fetch”;

const removedPrefixFromBucket = resolve(“.”);

/**
 * 
 * That method should be called before each S3 call to make sure to use a fresh token
 */
const readCredentials = async () => {
  const apiRepresentation =
    process.env.DATACENTER === “fake”
      ? JSON.parse(
          (
            await fs.readFile(resolve(“.”“fake-role-credentials.json”))
          ).toString()
        )
      : await fetch(
        /* replace iam-role with the role assumed by your api */
          “http://169.254.169.254/latest/meta-data/iam/security-credentials/iam-role”
        ).then((response=> response.json());
  return {
    accessKeyId: apiRepresentation.AccessKeyId,
    secretAccessKey: apiRepresentation.SecretAccessKey,
    sessionToken: apiRepresentation.Token,
  };
};

const getRemotePath = (pathPathLike=>
  path
    .toString()
    /* replace bucket-path with the directory where you want to synchronize data */
    .replace(removedPrefixFromBucket“/bucket-path”)
    .replace(/\\/g“/”)
    .replace(/\/+/g“/”);

/* replace with the right S3 bucket domain name */ 
const bucket = “the-bucket.s3-us-west-1.amazonaws.com”

export const remoteFs = {
  /**
   * reads the file if stored locally, otherwise 
   * fetches from S3 and saves a copy to the local storage
   * @param path path where to read the file
   * @param options encoding and flag (https://nodejs.org/api/fs.html#fs_file_system_flags)
   * @returns a promise to a buffer
   */
  readFile: async (
    pathPathLike,
    options?: {
      encoding?: null | undefined;
      flag?: OpenMode | undefined;
    } | null
  ): Promise<Buffer=> {
    let localFileErrorunknown = null;
    try {
      return await fs.readFile(pathoptions);
    } catch (e) {
      localFileError = e;
    }
    const credentials = bucket ? await readCredentials() : null;
    if (bucket && credentials) {
      const actualPath = getRemotePath(path);
      const signedParameters = new RequestSigner(
        {
          method: “GET”,
          headers: { Host: bucket },
          service: “s3”,
          path: actualPath,
        },
        credentials
      ).sign();
      const { headerssignedHeaders } = signedParameters;
      return await fetch(`https://${bucket}${actualPath}`, {
        headers: signedHeaders,
      }).then((response=> {
        if (response.statusCode !== 200)
          throw new Error(“file could not be downloaded”);
        return response.text().then((text=> {
          return fs.writeFile(pathtext!).then(() => Buffer.from(text!));
        }) as Promise<Buffer>;
      });
    }
    if (localFileError) {
      throw localFileError;
    } else throw new Error(`could not read file ${path}`);
  },

  /**
   * Updates mtime of the file
   * @param file file path
   * @returns a promise
   */
  touch: async (filePathLike): Promise<void=> {
    const credentials = bucket ? await readCredentials() : null;
    if (bucket && credentials) {
      const actualPath = getRemotePath(file);
      const source = `${bucket.split(“.”)[0]}${actualPath}`;
      const signedParameters = new RequestSigner(
        {
          method: “PUT”,
          headers: { Host: bucket“x-amz-copy-source”: source },
          service: “s3”,
          path: actualPath,
        },
        credentials
      ).sign();
      const { headerssignedHeaders } = signedParameters;
      (await fetch(`https://${bucket}${actualPath}`, {
        method: “PUT”,
        headers: { …signedHeaders“x-amz-copy-source”: source },
      }).then((response=> response.buffer())) as Buffer;
      return;
    }
  },

  /**
   * writes a file to the local storage and copy the  created file
   * to the S3 bucket
   * @param path path where to read the file
   * @param options encoding and flag (https://nodejs.org/api/fs.html#fs_file_system_flags)
   * @returns a promise to a buffer
   */
  writeFile: async (
    filePathLike,
    data:
      | string
      | NodeJS.ArrayBufferView
      | Iterable<string | NodeJS.ArrayBufferView>
      | AsyncIterable<string | NodeJS.ArrayBufferView>,
    options?:
      | (ObjectEncodingOptions & {
          mode?: Mode | undefined;
          flag?: OpenMode | undefined;
        })
      | BufferEncoding
      | null
  ): Promise<void=> {
    const credentials = bucket ? await readCredentials() : null;
    if (bucket && credentials) {
      const actualPath = getRemotePath(file);
      const signedParameters = new RequestSigner(
        {
          method: “PUT”,
          headers: { Host: bucket },
          service: “s3”,
          path: actualPath,
          body: data.toString(),
        },
        credentials
      ).sign();
      const { headerssignedHeaders } = signedParameters;
      (await fetch(`https://${bucket}${actualPath}`, {
        method: “PUT”,
        headers: signedHeaders,
        body: data.toString(),
      }).then((response=> response.buffer())) as Buffer;
      return;
    }
    return fs.writeFile(filedataoptions);
  },

  /**
   * Sends a file present in cache to the S3 bucket
   * @param file file path
   * @returns a promise
   */
  transmitFile: async (filePathLike): Promise<void=> {
    const credentials = bucket ? await readCredentials() : null;
    if (!bucket || !credentialsreturn;
    try {
      const data = await fs.readFile(file);
      await remoteFs.writeFile(filedata);
    } catch (e) {}
  },

  /**
   * Gets information about a file without reading the content
   * @param path a path to a file
   * @param opts options
   * @returns a promise to a Stat structure
   */
  stat: async (
    pathPathLike,
    opts?: StatOptions & {
      bigint?: false | undefined;
    }
  ): Promise<Stats=> {
    let localFileErrorunknown = null;
    try {
      return await fs.stat(pathopts);
    } catch (e) {
      localFileError = e;
    }
    const credentials = bucket ? await readCredentials() : null;
    if (bucket && credentials) {
      const actualPath = getRemotePath(path);
      const signedParameters = new RequestSigner(
        {
          method: “HEAD”,
          headers: { Host: bucket },
          service: “s3”,
          path: actualPath,
        },
        credentials
      ).sign();
      const { headerssignedHeaders } = signedParameters;
      return await fetch(`https://${bucket}${actualPath}`, {
        method: “HEAD”,
        headers: signedHeaders,
      }).then(
        (response=>
          ({
            isFile: () => response.statusCode === 200,
            mtime: new Date(
              response.headers[“last-modified”]?.toString() ?? “”
            ),
          } as Stats)
      );
    }
    if (localFileError) {
      throw localFileError;
    } else throw new Error(`could not stat file ${path}`);
  },
};

Using it locally without S3 profile automatically fallbacks to a simple filesystem storage, so you can still continue working with the module seamlessly without configuring your credentials to work with AWS.

Leave a Reply

Your email address will not be published. Required fields are marked *

*

This site uses Akismet to reduce spam. Learn how your comment data is processed.