diff --git a/src/account_x/__tests__/unit/containsFilter.unit.test.ts b/src/account_x/__tests__/unit/containsFilter.unit.test.ts new file mode 100644 index 00000000..9a7625ad --- /dev/null +++ b/src/account_x/__tests__/unit/containsFilter.unit.test.ts @@ -0,0 +1,145 @@ +import { describe, test, expect } from "vitest"; + +import { + containsFilterCondition, + containsFilterHasTooManyWildcards, + containsFilterMatches, + isEmptyContainsFilterText, +} from "../../controller/deletion/containsFilter"; + +describe("containsFilterMatches", () => { + test("matches a whole word regardless of case", () => { + expect(containsFilterMatches("Drinking coffee now", "Coffee")).toBe(true); + expect(containsFilterMatches("COFFEE", "coffee")).toBe(true); + }); + + test("matches the word inside a hashtag, mention, or punctuation", () => { + expect(containsFilterMatches("#Coffee - Fresh Cup", "Coffee")).toBe(true); + expect(containsFilterMatches("hey @Coffee!", "Coffee")).toBe(true); + expect(containsFilterMatches("(coffee)", "Coffee")).toBe(true); + expect(containsFilterMatches("#coffee\n\nnext line", "Coffee")).toBe(true); + }); + + test("does not match when the word is part of a longer word", () => { + expect(containsFilterMatches("@CoffeeBot hi", "Coffee")).toBe(false); + expect(containsFilterMatches("#CoffeeNow", "Coffee")).toBe(false); + expect(containsFilterMatches("uncoffee", "Coffee")).toBe(false); + }); + + test("a trailing star matches the rest of a word", () => { + const filter = "Coffee*"; + expect(containsFilterMatches("#Coffee", filter)).toBe(true); + expect(containsFilterMatches("@CoffeeBot", filter)).toBe(true); + expect(containsFilterMatches("Coffeebla", filter)).toBe(true); + expect(containsFilterMatches("coffeeTHIS", filter)).toBe(true); + expect(containsFilterMatches("cof fee", filter)).toBe(false); + }); + + test("a leading star matches the start of a word", () => { + expect(containsFilterMatches("the BestCoffee", "*Coffee")).toBe(true); + expect(containsFilterMatches("Coffee", "*Coffee")).toBe(true); + expect(containsFilterMatches("CoffeeBot", "*Coffee")).toBe(false); + }); + + test("a star in the middle matches letters, not spaces", () => { + expect(containsFilterMatches("Now-Playing now_bot", "now*bot")).toBe(true); + expect(containsFilterMatches("nowandbot", "now*bot")).toBe(true); + expect(containsFilterMatches("now and bot", "now*bot")).toBe(false); + }); + + test("a term that starts with # or @ needs no word edge before it", () => { + expect(containsFilterMatches("so#Coffee", "#Coffee")).toBe(true); + expect(containsFilterMatches("hey @someone", "@someone")).toBe(true); + expect(containsFilterMatches("hey @someoneelse", "@someone")).toBe(false); + }); + + test("takes everything else literally, including regex and SQL wildcards", () => { + expect(containsFilterMatches("it is 100% sure", "100%")).toBe(true); + expect(containsFilterMatches("it is 1000 sure", "100%")).toBe(false); + expect(containsFilterMatches("a_b", "a_b")).toBe(true); + expect(containsFilterMatches("axb", "a_b")).toBe(false); + expect(containsFilterMatches("cost $5 (approx)", "$5 (approx)")).toBe(true); + expect(containsFilterMatches("a.b", "a.b")).toBe(true); + expect(containsFilterMatches("axb", "a.b")).toBe(false); + expect(containsFilterMatches("back\\slash", "back\\slash")).toBe(true); + }); + + test("matches phrases with spaces", () => { + expect(containsFilterMatches("Hot Coffee: a cup", "hot coffee")).toBe(true); + expect(containsFilterMatches("Hot Coffee", "hot coffee")).toBe(false); + }); + + test("handles accented and non-latin words", () => { + expect(containsFilterMatches("Café del Mar", "café")).toBe(true); + expect(containsFilterMatches("Cafés", "café")).toBe(false); + expect(containsFilterMatches("音楽 を聴く", "音楽")).toBe(true); + }); + + test("ignores whitespace around the filter text", () => { + expect(containsFilterMatches("#Coffee", " Coffee ")).toBe(true); + }); + + test("never matches null text or an empty filter", () => { + expect(containsFilterMatches(null, "Coffee")).toBe(false); + expect(containsFilterMatches("anything", "")).toBe(false); + expect(containsFilterMatches("anything", " ")).toBe(false); + expect(containsFilterMatches("anything", "*")).toBe(false); + expect(containsFilterMatches("anything", " ** ")).toBe(false); + }); +}); + +describe("wildcard limit", () => { + test("stars in a row count as one", () => { + expect(containsFilterHasTooManyWildcards("a***b")).toBe(false); + expect(containsFilterMatches("axxb", "a***b")).toBe(true); + }); + + test("allows up to the limit and rejects more", () => { + expect(containsFilterHasTooManyWildcards("*a*b*c")).toBe(false); + expect(containsFilterHasTooManyWildcards("*a*b*c*")).toBe(true); + }); + + test("a filter over the limit matches nothing", () => { + expect(containsFilterMatches("abcde", "a*b*c*d*e")).toBe(false); + expect(containsFilterCondition("text", true, "a*b*c*d*e")).not.toBeNull(); + }); + + test("the worst case at the limit finishes quickly", () => { + const text = "a".repeat(300) + "c"; + const start = Date.now(); + expect(containsFilterMatches(text, "a*a*a*b")).toBe(false); + expect(Date.now() - start).toBeLessThan(1000); + }); +}); + +describe("isEmptyContainsFilterText", () => { + test("is empty for blanks and for stars alone", () => { + expect(isEmptyContainsFilterText("")).toBe(true); + expect(isEmptyContainsFilterText(" ")).toBe(true); + expect(isEmptyContainsFilterText("*")).toBe(true); + expect(isEmptyContainsFilterText(" * * ")).toBe(true); + }); + + test("is not empty when there is something to look for", () => { + expect(isEmptyContainsFilterText("a")).toBe(false); + expect(isEmptyContainsFilterText("*a*")).toBe(false); + expect(isEmptyContainsFilterText("#")).toBe(false); + }); +}); + +describe("containsFilterCondition", () => { + test("returns null when disabled or empty", () => { + expect(containsFilterCondition("text", false, "Coffee")).toBeNull(); + expect(containsFilterCondition("text", true, "")).toBeNull(); + expect(containsFilterCondition("text", true, " ")).toBeNull(); + expect(containsFilterCondition("text", true, "*")).toBeNull(); + }); + + test("binds the trimmed filter text for the given column", () => { + const condition = containsFilterCondition("t.text", true, " Now* "); + + expect(condition).not.toBeNull(); + expect(condition!.sql).toContain("t.text"); + expect(condition!.param).toBe("Now*"); + }); +}); diff --git a/src/account_x/__tests__/unit/controller.deletion.unit.test.ts b/src/account_x/__tests__/unit/controller.deletion.unit.test.ts index a4a47c1a..4e749a1c 100644 --- a/src/account_x/__tests__/unit/controller.deletion.unit.test.ts +++ b/src/account_x/__tests__/unit/controller.deletion.unit.test.ts @@ -97,6 +97,94 @@ describe("XAccountController - Deletion", () => { expect(ids).toEqual(["eligible"]); }); + + test("only returns tweets containing the filter text when enabled", async () => { + controller.account!.deleteTweetsFilterEnabled = true; + controller.account!.deleteTweetsFilterText = "@Someone"; + + seedTweet(controller, { + tweetID: "mentions", + text: "hey @someone look at this", + }); + seedTweet(controller, { + tweetID: "reply", + text: "@SOMEONE no way", + }); + seedTweet(controller, { + tweetID: "unrelated", + text: "nothing to see here", + }); + seedTweet(controller, { + tweetID: "retweet-of-them", + text: "RT @someone: their post", + }); + + const result = await controller.deleteTweetsStart(); + const ids = result.tweets.map((tweet) => tweet.id).sort(); + + expect(ids).toEqual(["mentions", "reply"]); + }); + + test("matches whole words only, and a star matches the rest of a word", async () => { + seedTweet(controller, { + tweetID: "hashtag", + text: "#Coffee - Fresh Cup", + }); + seedTweet(controller, { tweetID: "bot", text: "@CoffeeBot hello" }); + seedTweet(controller, { tweetID: "glued", text: "coffeeTHIS" }); + seedTweet(controller, { tweetID: "other", text: "nothing here" }); + + controller.account!.deleteTweetsFilterEnabled = true; + controller.account!.deleteTweetsFilterText = "Coffee"; + const whole = await controller.deleteTweetsStart(); + expect(whole.tweets.map((tweet) => tweet.id)).toEqual(["hashtag"]); + + controller.account!.deleteTweetsFilterText = "Coffee*"; + const starred = await controller.deleteTweetsStart(); + expect(starred.tweets.map((tweet) => tweet.id).sort()).toEqual([ + "bot", + "glued", + "hashtag", + ]); + }); + + test("does not filter at all for stars alone", async () => { + seedTweet(controller, { tweetID: "a", text: "one" }); + seedTweet(controller, { tweetID: "b", text: "two" }); + + controller.account!.deleteTweetsFilterEnabled = true; + controller.account!.deleteTweetsFilterText = "*"; + const result = await controller.deleteTweetsStart(); + + // The page never lets this through, so it must at least not be treated + // as a filter that matches a particular post + expect(result.tweets.length).toBe(2); + }); + + test("treats LIKE wildcards in the filter text literally", async () => { + controller.account!.deleteTweetsFilterEnabled = true; + controller.account!.deleteTweetsFilterText = "100%"; + + seedTweet(controller, { tweetID: "literal", text: "I am 100% sure" }); + seedTweet(controller, { tweetID: "not-literal", text: "100 percent" }); + + const result = await controller.deleteTweetsStart(); + const ids = result.tweets.map((tweet) => tweet.id); + + expect(ids).toEqual(["literal"]); + }); + + test("ignores the filter text when the filter is disabled", async () => { + controller.account!.deleteTweetsFilterEnabled = false; + controller.account!.deleteTweetsFilterText = "@someone"; + + seedTweet(controller, { tweetID: "a", text: "@someone hi" }); + seedTweet(controller, { tweetID: "b", text: "unrelated" }); + + const result = await controller.deleteTweetsStart(); + + expect(result.tweets.length).toBe(2); + }); }); describe("deleteRetweetsStart", () => { @@ -137,6 +225,33 @@ describe("XAccountController - Deletion", () => { expect(ids).toEqual(["old-retweet"]); }); + + test("only returns retweets containing the filter text when enabled", async () => { + controller.account!.deleteRetweetsFilterEnabled = true; + controller.account!.deleteRetweetsFilterText = "@someone"; + + seedTweet(controller, { + tweetID: "retweet-of-them", + text: "RT @Someone: their post", + }); + seedTweet(controller, { + tweetID: "retweet-mentioning-them", + text: "RT @other: cc @someone", + }); + seedTweet(controller, { + tweetID: "other-retweet", + text: "RT @other: unrelated", + }); + seedTweet(controller, { + tweetID: "regular-tweet", + text: "@someone not a retweet", + }); + + const result = await controller.deleteRetweetsStart(); + const ids = result.tweets.map((tweet) => tweet.id).sort(); + + expect(ids).toEqual(["retweet-mentioning-them", "retweet-of-them"]); + }); }); describe("deleteLikesStart", () => { @@ -277,5 +392,16 @@ describe("XAccountController - Deletion", () => { expect(count).toBe(1); }); + + test("applies the contains filter when total=false but not when total=true", async () => { + controller.account!.deleteTweetsFilterEnabled = true; + controller.account!.deleteTweetsFilterText = "butter"; + + seedTweet(controller, { tweetID: "match", text: "butter is great" }); + seedTweet(controller, { tweetID: "no-match", text: "margarine is fine" }); + + expect(await controller.deleteTweetsCountNotArchived(false)).toBe(1); + expect(await controller.deleteTweetsCountNotArchived(true)).toBe(2); + }); }); }); diff --git a/src/account_x/controller/deletion/containsFilter.ts b/src/account_x/controller/deletion/containsFilter.ts new file mode 100644 index 00000000..85aac2b1 --- /dev/null +++ b/src/account_x/controller/deletion/containsFilter.ts @@ -0,0 +1,112 @@ +// The "only if they contain ..." delete filter. +// +// The match is case-insensitive and on whole words, so "Coffee" matches +// "#Coffee" and "hot coffee: Coffee!" but not "CoffeeBot". A "*" +// stands for the rest of a word, so "Coffee*" also matches "CoffeeBot" +// and "coffeeTHIS". Everything else the user types is taken literally. +// +// SQLite's LIKE can't express word boundaries, so the match is a function +// registered on the account database (see registerContainsFilterFunction), and +// the counts, the preview and the delete all go through it. + +import type Database from "better-sqlite3"; + +export const CONTAINS_FILTER_FUNCTION = "cyd_text_matches"; + +const WORD_CHAR = "[\\p{L}\\p{N}_]"; +const WORD_CHAR_RE = new RegExp(WORD_CHAR, "u"); + +// Each "*" lets the pattern try every split of a word, so a handful of them +// against a long word can take effectively forever. Runs of stars count as one, +// and a filter with more than this many is rejected (it matches nothing). +export const MAX_CONTAINS_FILTER_WILDCARDS = 3; + +function collapseWildcards(text: string): string { + return text.trim().replace(/\*+/g, "*"); +} + +export function containsFilterHasTooManyWildcards(text: string): boolean { + const stars = collapseWildcards(text).match(/\*/g); + return (stars?.length ?? 0) > MAX_CONTAINS_FILTER_WILDCARDS; +} + +// True when there is nothing to look for. Stars alone don't count, since "*" +// would match every post. +export function isEmptyContainsFilterText(text: string): boolean { + return text.replace(/\*/g, "").trim() === ""; +} + +export function buildContainsFilterRegExp(text: string): RegExp { + const term = collapseWildcards(text); + const body = term + .split("*") + .map((part) => part.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")) + .join(`${WORD_CHAR}*`); + + // Only anchor to a word edge on a side that starts or ends with a word + // character, so "#Coffee" and "@someone" don't need a letter before them + const first = term[0]; + const last = term[term.length - 1]; + const before = WORD_CHAR_RE.test(first) ? `(? { + if (typeof text !== "string" || typeof filterText !== "string") { + return 0; + } + if (lastFilterText !== filterText || lastRegExp === null) { + if ( + isEmptyContainsFilterText(filterText) || + containsFilterHasTooManyWildcards(filterText) + ) { + return 0; + } + lastRegExp = buildContainsFilterRegExp(filterText); + lastFilterText = filterText; + } + return lastRegExp.test(text) ? 1 : 0; + }, + ); +} + +// Returns an SQL condition for `column` and the parameter it binds, or null +// when the filter is disabled or empty (which means "no filter"). +export function containsFilterCondition( + column: string, + enabled: boolean, + text: string, +): { sql: string; param: string } | null { + if (!enabled || isEmptyContainsFilterText(text)) { + return null; + } + return { + sql: `${CONTAINS_FILTER_FUNCTION}(${column}, ?) = 1`, + param: text.trim(), + }; +} diff --git a/src/account_x/controller/deletion/deleteRetweetsStart.ts b/src/account_x/controller/deletion/deleteRetweetsStart.ts index 69a5b1af..b7540148 100644 --- a/src/account_x/controller/deletion/deleteRetweetsStart.ts +++ b/src/account_x/controller/deletion/deleteRetweetsStart.ts @@ -2,6 +2,7 @@ import log from "electron-log/main"; import { getTimestampDaysAgo } from "../../../util"; import type { XAccountController } from "../../x_account_controller"; import type { XDeleteTweetsStartResponse } from "../../../shared_types"; +import { containsFilterCondition } from "./containsFilter"; // When you start deleting retweets, return a list of tweets to delete export async function deleteRetweetsStart( @@ -21,10 +22,21 @@ export async function deleteRetweetsStart( ? getTimestampDaysAgo(controller.account.deleteRetweetsDaysOld) : getTimestampDaysAgo(0); - const tweets = controller.fetchTweetsWithMediaAndURLs( - "t.deletedRetweetAt IS NULL AND t.text LIKE ? AND t.createdAt <= ?", - ["RT @%", daysOldTimestamp], + let whereClause = + "t.deletedRetweetAt IS NULL AND t.text LIKE ? AND t.createdAt <= ?"; + const params: (string | number)[] = ["RT @%", daysOldTimestamp]; + + const containsFilter = containsFilterCondition( + "t.text", + controller.account.deleteRetweetsFilterEnabled, + controller.account.deleteRetweetsFilterText, ); + if (containsFilter) { + whereClause += ` AND ${containsFilter.sql}`; + params.push(containsFilter.param); + } + + const tweets = controller.fetchTweetsWithMediaAndURLs(whereClause, params); return { tweets }; } diff --git a/src/account_x/controller/deletion/deleteTweetsCountNotArchived.ts b/src/account_x/controller/deletion/deleteTweetsCountNotArchived.ts index 01fbfcaa..f374113f 100644 --- a/src/account_x/controller/deletion/deleteTweetsCountNotArchived.ts +++ b/src/account_x/controller/deletion/deleteTweetsCountNotArchived.ts @@ -2,6 +2,7 @@ import log from "electron-log/main"; import { exec, Sqlite3Count } from "../../../database"; import { getTimestampDaysAgo } from "../../../util"; import type { XAccountController } from "../../x_account_controller"; +import { containsFilterCondition } from "./containsFilter"; // Returns the count of tweets that are not archived // If total is true, return the total count of tweets not archived @@ -20,80 +21,44 @@ export async function deleteTweetsCountNotArchived( throw new Error("Account not found"); } - // Select just the tweets that need to be deleted based on the settings - let count: Sqlite3Count; + // Count all non-deleted, non-archived tweets, with no filters + let whereClause = + "archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ?"; + const params: (string | number)[] = ["RT @%", controller.account.username]; - if (total) { - // Count all non-deleted, non-archived tweets, with no filters - count = exec( - controller.db, - "SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ?", - ["RT @%", controller.account.username], - "get", - ) as Sqlite3Count; - } else { + // Select just the tweets that need to be deleted based on the settings + if (!total) { const daysOldTimestamp = controller.account.deleteTweetsDaysOldEnabled ? getTimestampDaysAgo(controller.account.deleteTweetsDaysOld) : getTimestampDaysAgo(0); - if ( - controller.account.deleteTweetsLikesThresholdEnabled && - controller.account.deleteTweetsRetweetsThresholdEnabled - ) { - // Both likes and retweets thresholds - count = exec( - controller.db, - "SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ? AND retweetCount <= ?", - [ - "RT @%", - controller.account.username, - daysOldTimestamp, - controller.account.deleteTweetsLikesThreshold, - controller.account.deleteTweetsRetweetsThreshold, - ], - "get", - ) as Sqlite3Count; - } else if ( - controller.account.deleteTweetsLikesThresholdEnabled && - !controller.account.deleteTweetsRetweetsThresholdEnabled - ) { - // Just likes threshold - count = exec( - controller.db, - "SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND likeCount <= ?", - [ - "RT @%", - controller.account.username, - daysOldTimestamp, - controller.account.deleteTweetsLikesThreshold, - ], - "get", - ) as Sqlite3Count; - } else if ( - !controller.account.deleteTweetsLikesThresholdEnabled && - controller.account.deleteTweetsRetweetsThresholdEnabled - ) { - // Just retweets threshold - count = exec( - controller.db, - "SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ? AND retweetCount <= ?", - [ - "RT @%", - controller.account.username, - daysOldTimestamp, - controller.account.deleteTweetsRetweetsThreshold, - ], - "get", - ) as Sqlite3Count; - } else { - // Neither likes nor retweets threshold - count = exec( - controller.db, - "SELECT COUNT(*) AS count FROM tweet WHERE archivedAt IS NULL AND deletedTweetAt IS NULL AND text NOT LIKE ? AND username = ? AND createdAt <= ?", - ["RT @%", controller.account.username, daysOldTimestamp], - "get", - ) as Sqlite3Count; + whereClause += " AND createdAt <= ?"; + params.push(daysOldTimestamp); + + if (controller.account.deleteTweetsLikesThresholdEnabled) { + whereClause += " AND likeCount <= ?"; + params.push(controller.account.deleteTweetsLikesThreshold); + } + if (controller.account.deleteTweetsRetweetsThresholdEnabled) { + whereClause += " AND retweetCount <= ?"; + params.push(controller.account.deleteTweetsRetweetsThreshold); + } + const containsFilter = containsFilterCondition( + "text", + controller.account.deleteTweetsFilterEnabled, + controller.account.deleteTweetsFilterText, + ); + if (containsFilter) { + whereClause += ` AND ${containsFilter.sql}`; + params.push(containsFilter.param); } } + const count = exec( + controller.db, + `SELECT COUNT(*) AS count FROM tweet WHERE ${whereClause}`, + params, + "get", + ) as Sqlite3Count; + return count.count; } diff --git a/src/account_x/controller/deletion/deleteTweetsStart.ts b/src/account_x/controller/deletion/deleteTweetsStart.ts index 5e28983c..cb815168 100644 --- a/src/account_x/controller/deletion/deleteTweetsStart.ts +++ b/src/account_x/controller/deletion/deleteTweetsStart.ts @@ -2,6 +2,7 @@ import log from "electron-log/main"; import { getTimestampDaysAgo } from "../../../util"; import type { XAccountController } from "../../x_account_controller"; import type { XDeleteTweetsStartResponse } from "../../../shared_types"; +import { containsFilterCondition } from "./containsFilter"; export async function deleteTweetsStart( controller: XAccountController, @@ -42,6 +43,15 @@ export async function deleteTweetsStart( whereClause += " AND t.retweetCount <= ?"; params.push(controller.account.deleteTweetsRetweetsThreshold); } + const containsFilter = containsFilterCondition( + "t.text", + controller.account.deleteTweetsFilterEnabled, + controller.account.deleteTweetsFilterText, + ); + if (containsFilter) { + whereClause += ` AND ${containsFilter.sql}`; + params.push(containsFilter.param); + } // Fetch tweets using the helper function const tweets = controller.fetchTweetsWithMediaAndURLs(whereClause, params); diff --git a/src/account_x/controller/stats/getDeleteReviewStats.ts b/src/account_x/controller/stats/getDeleteReviewStats.ts index d4029252..6bb3f4f4 100644 --- a/src/account_x/controller/stats/getDeleteReviewStats.ts +++ b/src/account_x/controller/stats/getDeleteReviewStats.ts @@ -9,6 +9,9 @@ export async function getDeleteReviewStats( controller: XAccountController, ): Promise { const deleteReviewStats = emptyXDeleteReviewStats(); + // The delete settings may have just been saved from the options page, so + // count against what is in the database rather than the cached account + controller.refreshAccount(); if (!controller.account?.username) { log.info("XAccountController.getDeleteReviewStats: no account"); return deleteReviewStats; diff --git a/src/account_x/x_account_controller.ts b/src/account_x/x_account_controller.ts index 609343d3..ac99f844 100644 --- a/src/account_x/x_account_controller.ts +++ b/src/account_x/x_account_controller.ts @@ -58,6 +58,7 @@ import * as Jobs from "./controller/jobs"; import * as Account from "./controller/account"; import { fetchTweetsWithMediaAndURLsFromDB } from "./controller/fetchTweetsWithMediaAndURLs"; import { migrations } from "./controller/migrations"; +import { registerContainsFilterFunction } from "./controller/deletion/containsFilter"; import { BlueskyService } from "./controller/bluesky/BlueskyService"; import { sweepLegacyOAuthCredentials } from "../credentials"; import { migrateAccountBlueskyOAuthCredentials } from "../bluesky_oauth"; @@ -195,6 +196,7 @@ export class XAccountController extends BaseAccountController { log.info(`XAccountController.initDB: dbPath=${dbPath}`); this.db = new Database(dbPath, {}); this.db.pragma("journal_mode = WAL"); + registerContainsFilterFunction(this.db); runMigrations(this.db, migrations); // Older versions stored the X-to-Bluesky OAuth state and session in this diff --git a/src/database/migrations.ts b/src/database/migrations.ts index 14766ced..4c320020 100644 --- a/src/database/migrations.ts +++ b/src/database/migrations.ts @@ -279,6 +279,16 @@ SELECT id, type, sortOrder, xAccountId, uuid, facebookAccountID FROM account;`, `ALTER TABLE blueskyLocalAccount ADD COLUMN connectedAt DATETIME DEFAULT NULL;`, ], }, + // Add the "only if they contain ..." delete filters for tweets and retweets + { + name: "add deleteTweetsFilter and deleteRetweetsFilter to xAccount", + sql: [ + `ALTER TABLE xAccount ADD COLUMN deleteTweetsFilterEnabled BOOLEAN DEFAULT 0;`, + `ALTER TABLE xAccount ADD COLUMN deleteTweetsFilterText TEXT DEFAULT '';`, + `ALTER TABLE xAccount ADD COLUMN deleteRetweetsFilterEnabled BOOLEAN DEFAULT 0;`, + `ALTER TABLE xAccount ADD COLUMN deleteRetweetsFilterText TEXT DEFAULT '';`, + ], + }, ]; export const runMainMigrations = () => { diff --git a/src/database/x_account.ts b/src/database/x_account.ts index 4d8b32db..a1308d5a 100644 --- a/src/database/x_account.ts +++ b/src/database/x_account.ts @@ -31,6 +31,10 @@ interface XAccountRow { deleteRetweets: boolean; deleteRetweetsDaysOldEnabled: boolean; deleteRetweetsDaysOld: number; + deleteTweetsFilterEnabled: boolean; + deleteTweetsFilterText: string; + deleteRetweetsFilterEnabled: boolean; + deleteRetweetsFilterText: string; deleteLikes: boolean; deleteBookmarks: number; deleteDMs: boolean; @@ -81,6 +85,10 @@ function xAccountRowtoXAccount(row: XAccountRow): XAccount { deleteRetweets: !!row.deleteRetweets, deleteRetweetsDaysOldEnabled: !!row.deleteRetweetsDaysOldEnabled, deleteRetweetsDaysOld: row.deleteRetweetsDaysOld, + deleteTweetsFilterEnabled: !!row.deleteTweetsFilterEnabled, + deleteTweetsFilterText: row.deleteTweetsFilterText ?? "", + deleteRetweetsFilterEnabled: !!row.deleteRetweetsFilterEnabled, + deleteRetweetsFilterText: row.deleteRetweetsFilterText ?? "", deleteLikes: !!row.deleteLikes, deleteBookmarks: !!row.deleteBookmarks, deleteDMs: !!row.deleteDMs, @@ -176,6 +184,10 @@ export const saveXAccount = (account: XAccount) => { deleteRetweets = ?, deleteRetweetsDaysOldEnabled = ?, deleteRetweetsDaysOld = ?, + deleteTweetsFilterEnabled = ?, + deleteTweetsFilterText = ?, + deleteRetweetsFilterEnabled = ?, + deleteRetweetsFilterText = ?, deleteLikes = ?, deleteBookmarks = ?, deleteDMs = ?, @@ -220,6 +232,10 @@ export const saveXAccount = (account: XAccount) => { account.deleteRetweets ? 1 : 0, account.deleteRetweetsDaysOldEnabled ? 1 : 0, account.deleteRetweetsDaysOld, + account.deleteTweetsFilterEnabled ? 1 : 0, + account.deleteTweetsFilterText, + account.deleteRetweetsFilterEnabled ? 1 : 0, + account.deleteRetweetsFilterText, account.deleteLikes ? 1 : 0, account.deleteBookmarks ? 1 : 0, account.deleteDMs ? 1 : 0, diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 27caa520..95b7377b 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -339,6 +339,15 @@ "tweetsNotArchivedYet": "You have {strong} that haven't been archived yet", "orAtLeast": "or at least", "unlessTheyHaveAtLeast": "unless they have at least", + "onlyIfTheyContain": "only if they contain", + "containsFilterMissingText": "Type a word, {'@'}user or #hashtag to look for, or uncheck this box.", + "containsFilterTooManyWildcards": "That has too many * in it. Use {max} or fewer.", + "containsFilterWildcardHint": "Matches whole words only. Use * for the rest of a word, so Coffee* also finds CoffeeBot.", + "containsFilterPlaceholder": "{'@'}user, #hashtag or a word", + "containsFilterPreview": "{count} matches", + "containsFilterPreviewOne": "1 match", + "containsFilterPreviewNone": "nothing matches", + "containsFilterRateLimitNote": "X may pause the job for rate limits along the way; Cyd waits and continues on its own.", "likesDescription": "Likes are only visible to you on X. Cyd will delete all of the likes it can, but it can't delete ghost likes.", "bookmarksDescription": "Bookmarks are only visible to you on X.", "directMessagesWithdrawn": "X replaced direct messages with X Chat, and Cyd can't currently save or delete X Chat messages. Any direct messages Cyd already saved are still in your Cyd X archive, and you can still browse them.", @@ -390,6 +399,8 @@ "deleteTweetsUnlessRetweets": "Delete tweets unless they have at least a certain number of retweets", "deleteTweetsUnlessLikes": "Delete tweets unless they have at least a certain number of likes", "deleteRetweetsOlderThan": "Delete retweets older than a set number of days", + "deleteTweetsContaining": "Delete only tweets that contain certain text", + "deleteRetweetsContaining": "Delete only retweets that contain certain text", "unfollowEveryone": "Unfollow everyone", "deleteLikes": "Delete likes", "deleteBookmarks": "Delete bookmarks", @@ -520,6 +531,7 @@ "thatAreOlderThan": "that are older than {days} days", "unlessTheyHaveAtLeast": "unless they have at least {count} {type}", "unlessTheyHaveAtLeastOr": "unless they have at least {retweets} retweets or {likes} likes", + "thatContain": "that contain “{text}”", "haventSavedHTML": "You haven't saved HTML versions of any of these tweets.", "haventSavedHTMLSome": "You haven't saved HTML versions of {count} of these tweets.", "ifYouCare": "If you care,", diff --git a/src/renderer/src/test_util.ts b/src/renderer/src/test_util.ts index 1bb66020..ae23a261 100644 --- a/src/renderer/src/test_util.ts +++ b/src/renderer/src/test_util.ts @@ -55,6 +55,10 @@ export function createMockXAccount(overrides?: Partial): XAccount { deleteRetweets: false, deleteRetweetsDaysOldEnabled: false, deleteRetweetsDaysOld: 0, + deleteTweetsFilterEnabled: false, + deleteTweetsFilterText: "", + deleteRetweetsFilterEnabled: false, + deleteRetweetsFilterText: "", deleteLikes: false, deleteBookmarks: false, deleteDMs: false, diff --git a/src/renderer/src/util_x.test.ts b/src/renderer/src/util_x.test.ts index 380c4de0..aa1ceacd 100644 --- a/src/renderer/src/util_x.test.ts +++ b/src/renderer/src/util_x.test.ts @@ -53,6 +53,10 @@ function createXAccountFromDefaults(changes: object) { deleteRetweets: false, deleteRetweetsDaysOldEnabled: false, deleteRetweetsDaysOld: 0, + deleteTweetsFilterEnabled: false, + deleteTweetsFilterText: "", + deleteRetweetsFilterEnabled: false, + deleteRetweetsFilterText: "", deleteLikes: false, deleteBookmarks: false, unfollowEveryone: false, @@ -111,6 +115,8 @@ test("UtilX.xRequiresPremium() returns true when choosing any delete options", a "deleteTweetsLikesThresholdEnabled", "deleteTweetsRetweetsThresholdEnabled", "deleteRetweetsDaysOldEnabled", + "deleteTweetsFilterEnabled", + "deleteRetweetsFilterEnabled", "deleteLikes", "deleteBookmarks", "unfollowEveryone", diff --git a/src/renderer/src/util_x.ts b/src/renderer/src/util_x.ts index 4c2b220f..9826cebf 100644 --- a/src/renderer/src/util_x.ts +++ b/src/renderer/src/util_x.ts @@ -141,6 +141,11 @@ export async function xRequiresPremium( requiresPremium = true; } + if (xAccount.deleteTweets && xAccount.deleteTweetsFilterEnabled) { + console.log("Requires premium: deleteTweets and deleteTweetsFilterEnabled"); + requiresPremium = true; + } + // You can delete retweets for free, but only if you're not using these options if (xAccount.deleteRetweets && xAccount.deleteRetweetsDaysOldEnabled) { console.log( @@ -148,6 +153,12 @@ export async function xRequiresPremium( ); requiresPremium = true; } + if (xAccount.deleteRetweets && xAccount.deleteRetweetsFilterEnabled) { + console.log( + "Requires premium: deleteRetweets and deleteRetweetsFilterEnabled", + ); + requiresPremium = true; + } // Deleting likes, bookmarks, and unfollowing everyone are premium features if (xAccount.deleteLikes) { @@ -206,3 +217,19 @@ export async function xPostProgress( ); } } + +// True when an "only if they contain" filter has nothing to look for. Stars +// alone don't count, since "*" would match every post. Keep in step with +// isEmptyContainsFilterText in the main process. +export function isEmptyContainsFilterText(text: string): boolean { + return text.replace(/\*/g, "").trim() === ""; +} + +// More than this many "*" in a filter is rejected, since the match slows down +// badly with them. Keep in step with containsFilter.ts in the main process. +export const MAX_CONTAINS_FILTER_WILDCARDS = 3; + +export function containsFilterHasTooManyWildcards(text: string): boolean { + const stars = text.trim().replace(/\*+/g, "*").match(/\*/g); + return (stars?.length ?? 0) > MAX_CONTAINS_FILTER_WILDCARDS; +} diff --git a/src/renderer/src/view_models/XViewModel/auth.test.ts b/src/renderer/src/view_models/XViewModel/auth.test.ts index 16337185..d5f0743a 100644 --- a/src/renderer/src/view_models/XViewModel/auth.test.ts +++ b/src/renderer/src/view_models/XViewModel/auth.test.ts @@ -307,11 +307,17 @@ describe("auth.ts", () => { }); describe("loadUserStats", () => { - it("should show browser and automation notice", async () => { + it("should show the browser and keep the automation notice off while logging in", async () => { + // The notice blocks clicks, so it can't be on while the user may need to log in + let noticeDuringLogin: boolean | undefined; + mockVM.login = vi.fn().mockImplementation(async () => { + noticeDuringLogin = mockVM.showAutomationNotice; + }); + await Auth.loadUserStats(mockVM as XViewModel); expect(mockVM.showBrowser).toBe(true); - expect(mockVM.showAutomationNotice).toBe(true); + expect(noticeDuringLogin).toBe(false); }); it("should call login", async () => { diff --git a/src/renderer/src/view_models/XViewModel/auth.ts b/src/renderer/src/view_models/XViewModel/auth.ts index 74f70066..67818a8e 100644 --- a/src/renderer/src/view_models/XViewModel/auth.ts +++ b/src/renderer/src/view_models/XViewModel/auth.ts @@ -109,7 +109,9 @@ export async function login(vm: XViewModel): Promise { export async function loadUserStats(vm: XViewModel): Promise { vm.log("loadUserStats", "loading user stats"); vm.showBrowser = true; - vm.showAutomationNotice = true; + // login() shows the automation notice itself once login succeeds. Keep it off + // until then, otherwise someone who isn't logged in can't click the login form + vm.showAutomationNotice = false; vm.log("login", "getting user stats"); vm.instructions = vm.t("viewModels.x.auth.loadingStats"); diff --git a/src/renderer/src/view_models/XViewModel/view_model.test.ts b/src/renderer/src/view_models/XViewModel/view_model.test.ts index 70422335..b0ed1738 100644 --- a/src/renderer/src/view_models/XViewModel/view_model.test.ts +++ b/src/renderer/src/view_models/XViewModel/view_model.test.ts @@ -149,6 +149,10 @@ describe("XViewModel", () => { deleteRetweets: false, deleteRetweetsDaysOldEnabled: false, deleteRetweetsDaysOld: 0, + deleteTweetsFilterEnabled: false, + deleteTweetsFilterText: "", + deleteRetweetsFilterEnabled: false, + deleteRetweetsFilterText: "", deleteLikes: false, deleteBookmarks: false, deleteDMs: false, diff --git a/src/renderer/src/views/x/wizard/XWizardCheckPremium.vue b/src/renderer/src/views/x/wizard/XWizardCheckPremium.vue index c839f4a1..81461819 100644 --- a/src/renderer/src/views/x/wizard/XWizardCheckPremium.vue +++ b/src/renderer/src/views/x/wizard/XWizardCheckPremium.vue @@ -139,6 +139,36 @@ onMounted(async () => { +
  • +
    + + + {{ t("premium.deleteTweetsContaining") }} + +
    +
    • +
  • +
    + + + {{ t("premium.deleteRetweetsContaining") }} + +
    +
  • ({ setJobsType: vi.fn(), })); -vi.mock("../../../util_x", () => ({ - xHasSomeData: vi.fn().mockResolvedValue(true), -})); +vi.mock("../../../util_x", async () => { + const actual = + await vi.importActual("../../../util_x"); + return { + ...actual, + xHasSomeData: vi.fn().mockResolvedValue(true), + }; +}); // Mock child components vi.mock("../components/XLastImportOrBuildComponent.vue", () => ({ @@ -27,7 +32,7 @@ vi.mock("../components/XLastImportOrBuildComponent.vue", () => ({ vi.mock("../../shared_components/wizard/BaseWizardPage.vue", () => ({ default: { name: "BaseWizardPage", - template: "
    ", + template: '
    ', props: ["breadcrumbProps", "buttonProps"], }, })); @@ -43,6 +48,10 @@ describe("XWizardDeleteOptionsPage", () => { deleteTweets: false, deleteRetweets: false, deleteLikes: false, + deleteTweetsFilterEnabled: false, + deleteTweetsFilterText: "", + deleteRetweetsFilterEnabled: false, + deleteRetweetsFilterText: "", ...accountOverrides.xAccount, } as XAccount, }), @@ -118,6 +127,234 @@ describe("XWizardDeleteOptionsPage", () => { }); }); + describe("contains filter preview", () => { + const mountWithFilter = async (attachTo?: HTMLElement) => { + const mockModel = createMockModel({ + xAccount: { deleteTweets: true, deleteRetweets: true }, + }); + // The page loads its settings from the database, not from the model + window.electron.database.getAccount = vi + .fn() + .mockResolvedValue(mockModel.account); + wrapper = mount(XWizardDeleteOptionsPage, { + props: { model: mockModel as XViewModel }, + global: { plugins: [i18n] }, + attachTo, + }); + // Timers are faked in this block, so flush the mount's async work + // by advancing the fake clock rather than sleeping for real + await vi.advanceTimersByTimeAsync(50); + await wrapper.vm.$nextTick(); + // Open the "more options" for tweets + const showMore = wrapper + .findAll("button") + .find((btn) => btn.text().includes("Show more options")); + await showMore!.trigger("click"); + await wrapper.vm.$nextTick(); + }; + + beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + window.electron.X.getDeleteReviewStats = vi.fn().mockResolvedValue({ + tweetsToDelete: 59, + retweetsToDelete: 3, + likesToDelete: 0, + bookmarksToDelete: 0, + }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows no preview until a filter term is typed", async () => { + await mountWithFilter(); + + expect(wrapper.find("#deleteTweetsFilterPreview").exists()).toBe(false); + expect(window.electron.X.getDeleteReviewStats).not.toHaveBeenCalled(); + }); + + it("saves the filter and shows the match count after typing", async () => { + await mountWithFilter(); + + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + const textInput = wrapper.find("#deleteTweetsFilterText"); + await textInput.setValue("@someone"); + await textInput.trigger("input"); + + // Debounced: nothing fetched yet + expect(window.electron.X.getDeleteReviewStats).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(500); + await wrapper.vm.$nextTick(); + + expect(window.electron.database.saveAccount).toHaveBeenCalled(); + const saved = JSON.parse( + (window.electron.database.saveAccount as ReturnType).mock + .calls[0][0], + ); + expect(saved.xAccount.deleteTweetsFilterEnabled).toBe(true); + expect(saved.xAccount.deleteTweetsFilterText).toBe("@someone"); + expect(window.electron.X.getDeleteReviewStats).toHaveBeenCalledTimes(1); + + const preview = wrapper.find("#deleteTweetsFilterPreview"); + expect(preview.exists()).toBe(true); + expect(preview.text()).toContain("59 match"); + }); + + const continueDisabled = () => + wrapper.findComponent({ name: "BaseWizardPage" }).props("buttonProps") + .nextButtons[0].disabled; + + it("loads a saved filter that is ticked but empty as unticked", async () => { + const mockModel = createMockModel({ + xAccount: { + deleteTweets: true, + deleteRetweets: true, + deleteTweetsFilterEnabled: true, + deleteTweetsFilterText: "", + deleteRetweetsFilterEnabled: true, + deleteRetweetsFilterText: " ", + }, + }); + window.electron.database.getAccount = vi + .fn() + .mockResolvedValue(mockModel.account); + wrapper = mount(XWizardDeleteOptionsPage, { + props: { model: mockModel as XViewModel }, + global: { plugins: [i18n] }, + }); + await vi.advanceTimersByTimeAsync(50); + await wrapper.vm.$nextTick(); + + // Both "more options" sections are closed, since no filter is active + const showMoreButtons = wrapper + .findAll("button") + .filter((btn) => btn.text().includes("Show more options")); + for (const btn of showMoreButtons) { + await btn.trigger("click"); + } + await wrapper.vm.$nextTick(); + + expect( + (wrapper.find("#deleteTweetsFilterEnabled").element as HTMLInputElement) + .checked, + ).toBe(false); + expect( + ( + wrapper.find("#deleteRetweetsFilterEnabled") + .element as HTMLInputElement + ).checked, + ).toBe(false); + expect(continueDisabled()).toBe(false); + }); + + it("blocks Continue while a ticked tweets filter has no text", async () => { + await mountWithFilter(); + expect(continueDisabled()).toBe(false); + + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + expect(continueDisabled()).toBe(true); + expect(wrapper.find("#deleteTweetsFilterMissing").exists()).toBe(true); + + await wrapper.find("#deleteTweetsFilterText").setValue(" "); + expect(continueDisabled()).toBe(true); + + await wrapper.find("#deleteTweetsFilterText").setValue("@someone"); + expect(continueDisabled()).toBe(false); + expect(wrapper.find("#deleteTweetsFilterMissing").exists()).toBe(false); + }); + + it("blocks Continue when a filter has too many wildcards", async () => { + await mountWithFilter(); + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + + await wrapper.find("#deleteTweetsFilterText").setValue("a*b*c*d*e"); + expect(continueDisabled()).toBe(true); + expect(wrapper.find("#deleteTweetsFilterTooManyWildcards").exists()).toBe( + true, + ); + expect(wrapper.find("#deleteTweetsFilterPreview").exists()).toBe(false); + + // Stars in a row count once + await wrapper.find("#deleteTweetsFilterText").setValue("a***b*c*d"); + expect(continueDisabled()).toBe(false); + expect(wrapper.find("#deleteTweetsFilterTooManyWildcards").exists()).toBe( + false, + ); + }); + + it("unticks the filter when its text is cleared", async () => { + await mountWithFilter(); + const checkbox = wrapper.find("#deleteTweetsFilterEnabled"); + const text = wrapper.find("#deleteTweetsFilterText"); + + await checkbox.setValue(true); + await text.setValue("@someone"); + expect((checkbox.element as HTMLInputElement).checked).toBe(true); + + await text.setValue(""); + expect((checkbox.element as HTMLInputElement).checked).toBe(false); + expect(continueDisabled()).toBe(false); + }); + + it("puts the cursor in the text box when the filter is ticked", async () => { + const root = document.createElement("div"); + document.body.appendChild(root); + try { + await mountWithFilter(root); + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + await wrapper.vm.$nextTick(); + + expect(document.activeElement?.id).toBe("deleteTweetsFilterText"); + } finally { + wrapper.unmount(); + root.remove(); + } + }); + + it("lets Continue through again when the empty filter is unticked", async () => { + await mountWithFilter(); + + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + expect(continueDisabled()).toBe(true); + + await wrapper.find("#deleteTweetsFilterEnabled").setValue(false); + expect(continueDisabled()).toBe(false); + }); + + it("blocks Continue while a ticked retweets filter has no text", async () => { + await mountWithFilter(); + const showMoreButtons = wrapper + .findAll("button") + .filter((btn) => btn.text().includes("Show more options")); + await showMoreButtons[0].trigger("click"); + await wrapper.vm.$nextTick(); + + await wrapper.find("#deleteRetweetsFilterEnabled").setValue(true); + expect(continueDisabled()).toBe(true); + expect(wrapper.find("#deleteRetweetsFilterMissing").exists()).toBe(true); + + await wrapper.find("#deleteRetweetsFilterText").setValue("#hashtag"); + expect(continueDisabled()).toBe(false); + }); + + it("only fetches once for a burst of keystrokes", async () => { + await mountWithFilter(); + + await wrapper.find("#deleteTweetsFilterEnabled").setValue(true); + const textInput = wrapper.find("#deleteTweetsFilterText"); + for (const value of ["@", "@s", "@so"]) { + await textInput.setValue(value); + await textInput.trigger("input"); + await vi.advanceTimersByTimeAsync(100); + } + await vi.advanceTimersByTimeAsync(500); + + expect(window.electron.X.getDeleteReviewStats).toHaveBeenCalledTimes(1); + }); + }); + describe("data loading", () => { it("should check for existing data on mount", async () => { const { xHasSomeData } = await import("../../../util_x"); diff --git a/src/renderer/src/views/x/wizard/XWizardDeleteOptionsPage.vue b/src/renderer/src/views/x/wizard/XWizardDeleteOptionsPage.vue index 3ceb41fb..caa1a70a 100644 --- a/src/renderer/src/views/x/wizard/XWizardDeleteOptionsPage.vue +++ b/src/renderer/src/views/x/wizard/XWizardDeleteOptionsPage.vue @@ -1,8 +1,13 @@