JSON archivátor

.gitignore

@@ -11,3 +11,4 @@ config/assets
 config/lists
 pages
 logs
+archive

package.json

@@ -15,7 +15,7 @@
     "css": "npx sass ./web/sass:./public/css --no-source-map --style compressed && npx postcss ./public/css/**/*.css --use autoprefixer --replace",
     "map-assets": "node ./dist/scripts/map_assets.js",
     "build": "npm run backend && npm run js && npm run css && npm run map-assets",
-    "dirs": "mkdir -p ./public/{js,css,uploads,banners,captcha} ./public/uploads/thumb ./config/lists ./{pages,logs}",
+    "dirs": "mkdir -p ./public/{js,css,uploads,banners,captcha} ./public/uploads/thumb ./config/lists ./{pages,logs,archive} ./archive/files",
     "configure": "node ./dist/scripts/configure.js",
     "create-admin": "node ./dist/scripts/create_admin.js",
     "get-service-lists": "node ./dist/scripts/get_service_lists.js",

src/db/post.ts

@@ -1,6 +1,7 @@
 import { redis } from "../cache.js";
 import type { CzchanConfig } from "../config.js";
 import { postgres } from "../db/client.js";
+import { archiveThread } from "../lib/archive.js";
 import { CzchanError } from "../lib/error.js";
 import {
   postQuote,
@@ -508,8 +509,14 @@ const slideThreads = async (config: CzchanConfig, board: Board) => {
     config.site.ui.page_size * board.config.pages,
   );
 
-  for (const thread of slidThreads) {
-    await deletePost(thread);
+  for (const op of slidThreads) {
+    // Archive if enabled
+    if (board.config.archive) {
+      const replies = await readPostReplies(op);
+      await archiveThread(board, op, replies);
+    }
+
+    await deletePost(op);
   }
 };
 
@@ -549,7 +556,7 @@ const deletePostFile = async (post: Post, hash: string) => {
     await removeUsageFromArray(record, `post;${post.board};${post.id}`);
   }
 
-  // Mark as deleted
+  // Mark as deleted (if not already)
   const files = post.files.map((file) =>
     file.hash === hash
       ? {
@@ -565,15 +572,16 @@ const deletePostFile = async (post: Post, hash: string) => {
 // Deletes all files from a post
 const deletePostFiles = async (post: Post) => {
   const hashes = post.files.map((file) => file.hash);
+
   const records = await readFileRecords(hashes);
 
   for (const record of records) {
     await removeUsageFromArray(record, `post;${post.board};${post.id}`);
   }
 
-  const files = post.files.map((file) => ({ ...file, deleted: true }));
+  const files = post.files.map((_) => ({ deleted: true }));
 
-  await updatePostFiles(post, files);
+  await updatePostFiles(post, files as File[]);
 };
 
 // Universal abstraction

src/lib/archive.ts

@@ -0,0 +1,225 @@
+import type { Board, Post } from "../schema/tables.js";
+import { existsSync } from "node:fs";
+import { writeFile, copyFile } from "node:fs/promises";
+import { join, parse } from "node:path";
+
+// Only archive what's displayed
+type ArchivedPost = {
+  // Identifiers
+  board: string;
+  id: number;
+  thread: number | null;
+  // Header
+  subject: string | null;
+  name: string | null;
+  tripcode: string | null;
+  capcode: string | null;
+  email: string | null;
+  // Optional
+  user_id: string | null; // User ID (if visible)
+  geo_flag: string | null; // Country (if visible)
+  // Numbers
+  t_bumps: number;
+  t_replies: number;
+  t_files: number;
+  // Attributes
+  f_sticky: number;
+  f_locked: boolean;
+  f_bumplocked: boolean;
+  f_looped: boolean;
+  // Content
+  content: string;
+  content_unformatted: string;
+  files: ArchivedFile[];
+  // Timestamps
+  bumped: number;
+  created: number;
+};
+
+// Likewise for files
+type ArchivedFile =
+  | { deleted: true }
+  | {
+      deleted: false;
+      original_filename: string;
+      type: string;
+      format: string;
+      size: number;
+      dimensions: [number, number] | null;
+      duration: number | null;
+      hash: string;
+      phash: string;
+      // Paths (relative to JSON/HTML)
+      url: string;
+      path: string;
+      thumb_url: string | null;
+      thumb_path: string | null;
+      spoiler: boolean;
+    };
+
+// Find quotes and parse them
+const QUOTE_HTML_REGEX = /<a class="quote" href=".+?">(.+?)<\/a>/g;
+
+const archiveThread = async (board: Board, op: Post, replies: Post[]) => {
+  const posts = [op, ...replies];
+
+  const allowedQuotes = new Set(
+    posts.map((post) => `${post.board};${post.id}`),
+  );
+
+  const archivedPosts = [];
+
+  for (const post of posts) {
+    const archivedPost = await archivePost(board, post, allowedQuotes);
+
+    archivedPosts.push(archivedPost);
+  }
+
+  const archivePath = join(".", "archive", `${board.id}-${op.id}.json`);
+  const json = JSON.stringify(archivedPosts, null, 2);
+
+  await writeFile(archivePath, json);
+};
+
+// Modify post and copy files
+const archivePost = async (
+  board: Board,
+  post: Post,
+  allowedQuotes: Set<string>,
+): Promise<ArchivedPost> => {
+  const newFiles: ArchivedFile[] = [];
+
+  for (let i = 0; i < post.files.length; i++) {
+    const oldFile = post.files[i];
+
+    if (oldFile.deleted) {
+      newFiles.push({ deleted: true });
+      continue;
+    }
+
+    // Copy file
+    const fileHash = oldFile.hash;
+    const filePath = oldFile.path;
+    const fileFormat = parse(filePath).ext;
+    const newFilePath = join(
+      ".",
+      "archive",
+      "files",
+      `${fileHash}${fileFormat}`,
+    );
+
+    const newFileURL = join(".", "files", `${fileHash}${fileFormat}`);
+
+    if (!existsSync(newFilePath)) {
+      await copyFile(filePath, newFilePath);
+    }
+
+    // Copy thumbnails
+    const thumbPath = oldFile.thumb_path;
+
+    let newThumbPath = null;
+    let newThumbURL = null;
+
+    if (thumbPath) {
+      const thumbFormat = parse(thumbPath).ext;
+
+      newThumbPath = join(
+        ".",
+        "archive",
+        "files",
+        `thumb-${fileHash}${thumbFormat}`,
+      );
+
+      newThumbURL = join(".", "files", `thumb-${fileHash}${thumbFormat}`);
+
+      if (!existsSync(newThumbURL)) {
+        await copyFile(thumbPath, newThumbPath);
+      }
+    }
+
+    // Push the new file
+
+    const newFile = {
+      ...oldFile,
+      url: newFileURL,
+      path: newFilePath,
+      thumb_path: newThumbPath,
+      thumb_url: newThumbURL,
+    };
+
+    newFiles.push(newFile);
+  }
+
+  const newContent = fixContent(board.id, post.content, allowedQuotes);
+
+  const newPost = {
+    // Copy
+    board: post.board,
+    id: post.id,
+    thread: post.thread,
+    subject: post.subject,
+    name: post.name,
+    tripcode: post.tripcode,
+    capcode: post.capcode,
+    email: post.email,
+    t_bumps: post.t_bumps,
+    t_replies: post.t_replies,
+    t_files: post.t_files,
+    f_sticky: post.f_sticky,
+    f_locked: post.f_locked,
+    f_bumplocked: post.f_bumplocked,
+    f_looped: post.f_looped,
+    content_unformatted: post.content_unformatted,
+    // Update
+    user_id: board.config.user_ids ? post.user_id : null,
+    geo_flag: board.config.geo_flags ? (post.metadata.country ?? "xx") : null,
+    content: newContent,
+    files: newFiles,
+    bumped: post.bumped.getTime(),
+    created: post.created.getTime(),
+  };
+
+  return newPost;
+};
+
+const fixContent = (
+  board: string,
+  text: string,
+  allowedQuotes: Set<string>,
+) => {
+  text = text.replace(QUOTE_HTML_REGEX, (match, ...args: any[]) => {
+    const quote = parseQuote(board, args[args.length - 1]);
+
+    if (allowedQuotes.has(quote)) {
+      return match;
+    } else {
+      return `<s class="dead-link">${args[args.length - 1]} (MIMO VLÁKNO)</s>`;
+    }
+  });
+
+  return text;
+};
+
+const parseQuote = (board: string | null, text: string) => {
+  const quote = text.split(" ")[0];
+  const isCross = quote.startsWith("&gt;&gt;&gt;");
+
+  if (isCross) {
+    const segments = quote.split("/");
+    const board = segments[1];
+    const id = segments[2];
+
+    // Board link, not a quote
+    if (!id) {
+      return board;
+    }
+
+    return `${board};${id}`;
+  } else {
+    const segments = quote.split("&gt;&gt;");
+    const id = segments[1];
+    return `${board};${id}`;
+  }
+};
+
+export { archiveThread };

src/lib/file.ts

@@ -209,20 +209,7 @@ const processFile = async (
   thumbs: boolean,
 ): Promise<File> => {
   // TODO: phash
-  const stream = createReadStream(file.currentFile);
-  const sha256 = createHash("sha256");
-  sha256.setEncoding("hex");
-
-  const hash: string = await new Promise((resolve, reject) => {
-    stream.on("end", () => {
-      sha256.end();
-      resolve(sha256.read());
-    });
-
-    stream.on("error", reject);
-    stream.pipe(sha256);
-  });
-
+  const hash = await hashFile(file.currentFile);
   const phash = ""; // Shut up
 
   // Deduplicate
@@ -627,4 +614,23 @@ const tryThumbnailAudio = async (
   }
 };
 
-export { processFiles };
+// Helper
+const hashFile = async (path: string) => {
+  const stream = createReadStream(path);
+  const sha256 = createHash("sha256");
+  sha256.setEncoding("hex");
+
+  const hash: string = await new Promise((resolve, reject) => {
+    stream.on("end", () => {
+      sha256.end();
+      resolve(sha256.read());
+    });
+
+    stream.on("error", reject);
+    stream.pipe(sha256);
+  });
+
+  return hash;
+};
+
+export { processFiles, hashFile };

src/web/components/content/thumbnail.tsx

@@ -6,14 +6,16 @@ const Thumbnail = ({
   nospoiler = false,
 }: PropsWithChildren<{ file: File; nospoiler?: boolean }>) => (
   <img
-    class={`thumb thumb-${file.type}`}
+    class={`thumb${file.deleted ? "" : ` thumb-${file.type}`}`}
     src={(() => {
       if (file.deleted) {
         return "/public/img/thumb/deleted.png";
       }
+
       if (!nospoiler && file.spoiler) {
         return "/public/img/thumb/spoiler.png";
       }
+
       if (file.thumb_url) {
         return file.thumb_url;
       }

web/ts/lib/util.ts

@@ -34,6 +34,12 @@ const parseQuote = (board: string | null, text: string) => {
     const segments = quote.split("/");
     const board = segments[1];
     const id = segments[2];
+
+    // Board link, not a quote
+    if (!id) {
+      return null;
+    }
+
     return `${board}-${id}`;
   } else {
     const segments = quote.split(">>");

web/ts/posts/quote/hover_quote.ts

@@ -43,6 +43,11 @@ const hoverQuote = (quotes: JQuery<HTMLElement>) => {
       quote.closest(".post, .catalog-tile").attr("data-board") ?? null;
 
     const id = parseQuote(board, quote.text());
+
+    if (!id) {
+      return;
+    }
+
     const existingPost = $(`#${id}.post`); // Catalog tiles don't count
     const url = quote.attr("href");
 

web/ts/posts/yous.ts

@@ -32,6 +32,11 @@ const addYous = (posts: JQuery<HTMLElement>) => {
     quotes.each(function () {
       const quote = $(this).text();
       const parsed = parseQuote(board, quote);
+
+      if (!parsed) {
+        return;
+      }
+
       const isYou = yous.includes(parsed);
       if (isYou) {
         $(this).append(" (Ty)");