♻️ (reports) unify selectedCategories and conditions (#3178)

packages/desktop-client/src/components/reports/ReportSidebar.tsx

@@ -8,6 +8,7 @@ import { type LocalPrefs } from 'loot-core/types/prefs';
 
 import { styles } from '../../style/styles';
 import { theme } from '../../style/theme';
+import { Information } from '../alerts';
 import { Button } from '../common/Button';
 import { Menu } from '../common/Menu';
 import { Popover } from '../common/Popover';
@@ -26,6 +27,7 @@ import { setSessionReport } from './setSessionReport';
 
 type ReportSidebarProps = {
   customReportItems: CustomReportEntity;
+  selectedCategories: CategoryEntity[];
   categories: { list: CategoryEntity[]; grouped: CategoryGroupEntity[] };
   dateRangeLine: number;
   allIntervals: { name: string; pretty: string }[];
@@ -55,10 +57,12 @@ type ReportSidebarProps = {
   defaultModeItems: (graph: string, item: string) => void;
   earliestTransaction: string;
   firstDayOfWeekIdx: LocalPrefs['firstDayOfWeekIdx'];
+  isComplexCategoryCondition?: boolean;
 };
 
 export function ReportSidebar({
   customReportItems,
+  selectedCategories,
   categories,
   dateRangeLine,
   allIntervals,
@@ -82,6 +86,7 @@ export function ReportSidebar({
   defaultModeItems,
   earliestTransaction,
   firstDayOfWeekIdx,
+  isComplexCategoryCondition = false,
 }: ReportSidebarProps) {
   const [menuOpen, setMenuOpen] = useState(false);
   const triggerRef = useRef(null);
@@ -536,19 +541,25 @@ export function ReportSidebar({
           minHeight: 200,
         }}
       >
-        <CategorySelector
-          categoryGroups={categories.grouped.filter(f => {
-            return customReportItems.showHiddenCategories || !f.hidden
-              ? true
-              : false;
-          })}
-          selectedCategories={customReportItems.selectedCategories || []}
-          setSelectedCategories={e => {
-            setSelectedCategories(e);
-            onReportChange({ type: 'modify' });
-          }}
-          showHiddenCategories={customReportItems.showHiddenCategories}
-        />
+        {isComplexCategoryCondition ? (
+          <Information>
+            Remove active category filters to show the category selector.
+          </Information>
+        ) : (
+          <CategorySelector
+            categoryGroups={categories.grouped.filter(f => {
+              return customReportItems.showHiddenCategories || !f.hidden
+                ? true
+                : false;
+            })}
+            selectedCategories={selectedCategories || []}
+            setSelectedCategories={e => {
+              setSelectedCategories(e);
+              onReportChange({ type: 'modify' });
+            }}
+            showHiddenCategories={customReportItems.showHiddenCategories}
+          />
+        )}
       </View>
     </View>
   );

packages/desktop-client/src/components/reports/reports/CustomReport.tsx

@@ -55,6 +55,49 @@ import { createGroupedSpreadsheet } from '../spreadsheets/grouped-spreadsheet';
 import { useReport } from '../useReport';
 import { fromDateRepr } from '../util';
 
+/**
+ * Transform `selectedCategories` into `conditions`.
+ */
+function useSelectedCategories(
+  conditions: RuleConditionEntity[],
+  categories: CategoryEntity[],
+): CategoryEntity[] {
+  const existingCategoryCondition = useMemo(
+    () => conditions.find(({ field }) => field === 'category'),
+    [conditions],
+  );
+
+  return useMemo(() => {
+    if (!existingCategoryCondition) {
+      return categories;
+    }
+
+    switch (existingCategoryCondition.op) {
+      case 'is':
+        return categories.filter(
+          ({ id }) => id === existingCategoryCondition.value,
+        );
+
+      case 'isNot':
+        return categories.filter(
+          ({ id }) => existingCategoryCondition.value !== id,
+        );
+
+      case 'oneOf':
+        return categories.filter(({ id }) =>
+          existingCategoryCondition.value.includes(id),
+        );
+
+      case 'notOneOf':
+        return categories.filter(
+          ({ id }) => !existingCategoryCondition.value.includes(id),
+        );
+    }
+
+    return categories;
+  }, [existingCategoryCondition, categories]);
+}
+
 export function CustomReport() {
   const categories = useCategories();
   const { isNarrowWidth } = useResponsive();
@@ -102,9 +145,65 @@ export function CustomReport() {
     }>
   >([]);
 
-  const [selectedCategories, setSelectedCategories] = useState(
-    loadReport.selectedCategories,
-  );
+  // Complex category conditions are:
+  // - conditions with multiple "category" fields
+  // - conditions with "category" field that use "contains", "doesNotContain" or "matches" operations
+  const isComplexCategoryCondition =
+    !!conditions.find(
+      ({ field, op }) =>
+        field === 'category' &&
+        ['contains', 'doesNotContain', 'matches'].includes(op),
+    ) || conditions.filter(({ field }) => field === 'category').length >= 2;
+
+  const setSelectedCategories = (newCategories: CategoryEntity[]) => {
+    const newCategoryIdSet = new Set(newCategories.map(({ id }) => id));
+    const allCategoryIds = categories.list.map(({ id }) => id);
+    const allCategoriesSelected = !allCategoryIds.find(
+      id => !newCategoryIdSet.has(id),
+    );
+    const newCondition = {
+      field: 'category',
+      op: 'oneOf',
+      value: newCategories.map(({ id }) => id),
+      type: 'id',
+    } satisfies RuleConditionEntity;
+
+    const existingCategoryCondition = conditions.find(
+      ({ field }) => field === 'category',
+    );
+
+    // If the existing conditions already have one for "category" - replace it
+    if (existingCategoryCondition) {
+      // If we selected all categories - remove the filter (default state)
+      if (allCategoriesSelected) {
+        onDeleteFilter(existingCategoryCondition);
+        return;
+      }
+
+      // Update the "notOneOf" condition if it's already set
+      if (existingCategoryCondition.op === 'notOneOf') {
+        onUpdateFilter(existingCategoryCondition, {
+          ...existingCategoryCondition,
+          value: allCategoryIds.filter(id => !newCategoryIdSet.has(id)),
+        });
+        return;
+      }
+
+      // Otherwise use `oneOf` condition
+      onUpdateFilter(existingCategoryCondition, newCondition);
+      return;
+    }
+
+    // Don't add a new filter if all categories are selected (default state)
+    if (allCategoriesSelected) {
+      return;
+    }
+
+    // If the existing conditions does not have a "category" - append a new one
+    onApplyFilter(newCondition);
+  };
+
+  const selectedCategories = useSelectedCategories(conditions, categories.list);
   const [startDate, setStartDate] = useState(loadReport.startDate);
   const [endDate, setEndDate] = useState(loadReport.endDate);
   const [mode, setMode] = useState(loadReport.mode);
@@ -146,12 +245,6 @@ export function CustomReport() {
       : loadReport.savedStatus ?? 'new',
   );
 
-  useEffect(() => {
-    if (selectedCategories === undefined && categories.list.length !== 0) {
-      setSelectedCategories(categories.list);
-    }
-  }, [categories, selectedCategories]);
-
   useEffect(() => {
     async function run() {
       onApplyFilter(null);
@@ -260,7 +353,6 @@ export function CustomReport() {
       endDate,
       interval,
       categories,
-      selectedCategories,
       conditions,
       conditionsOp,
       showEmpty,
@@ -276,7 +368,6 @@ export function CustomReport() {
     interval,
     balanceTypeOp,
     categories,
-    selectedCategories,
     conditions,
     conditionsOp,
     showEmpty,
@@ -293,7 +384,6 @@ export function CustomReport() {
       endDate,
       interval,
       categories,
-      selectedCategories,
       conditions,
       conditionsOp,
       showEmpty,
@@ -315,7 +405,6 @@ export function CustomReport() {
     groupBy,
     balanceTypeOp,
     categories,
-    selectedCategories,
     payees,
     accounts,
     conditions,
@@ -348,7 +437,6 @@ export function CustomReport() {
     showHiddenCategories,
     includeCurrentInterval,
     showUncategorized,
-    selectedCategories,
     graphType,
     conditions,
     conditionsOp,
@@ -471,13 +559,6 @@ export function CustomReport() {
   };
 
   const setReportData = (input: CustomReportEntity) => {
-    const selectAll: CategoryEntity[] = [];
-    categories.grouped.map(categoryGroup =>
-      (categoryGroup.categories || []).map(category =>
-        selectAll.push(category),
-      ),
-    );
-
     setStartDate(input.startDate);
     setEndDate(input.endDate);
     setIsDateStatic(input.isDateStatic);
@@ -491,7 +572,6 @@ export function CustomReport() {
     setShowHiddenCategories(input.showHiddenCategories);
     setIncludeCurrentInterval(input.includeCurrentInterval);
     setShowUncategorized(input.showUncategorized);
-    setSelectedCategories(input.selectedCategories || selectAll);
     setGraphType(input.graphType);
     onApplyFilter(null);
     (input.conditions || []).forEach(condition => onApplyFilter(condition));
@@ -578,6 +658,7 @@ export function CustomReport() {
         {!isNarrowWidth && (
           <ReportSidebar
             customReportItems={customReportItems}
+            selectedCategories={selectedCategories}
             categories={categories}
             dateRangeLine={dateRangeLine}
             allIntervals={allIntervals}
@@ -601,6 +682,7 @@ export function CustomReport() {
             defaultModeItems={defaultModeItems}
             earliestTransaction={earliestTransaction}
             firstDayOfWeekIdx={firstDayOfWeekIdx}
+            isComplexCategoryCondition={isComplexCategoryCondition}
           />
         )}
         <View

packages/desktop-client/src/components/reports/reports/GetCardData.tsx

@@ -114,7 +114,6 @@ export function GetCardData({
       endDate,
       interval: report.interval,
       categories,
-      selectedCategories: report.selectedCategories ?? categories.list,
       conditions: report.conditions ?? [],
       conditionsOp: report.conditionsOp,
       showEmpty: report.showEmpty,
@@ -131,7 +130,6 @@ export function GetCardData({
       endDate,
       interval: report.interval,
       categories,
-      selectedCategories: report.selectedCategories ?? categories.list,
       conditions: report.conditions ?? [],
       conditionsOp: report.conditionsOp,
       showEmpty: report.showEmpty,

packages/desktop-client/src/components/reports/reports/Spending.tsx

@@ -4,7 +4,6 @@ import * as monthUtils from 'loot-core/src/shared/months';
 import { amountToCurrency } from 'loot-core/src/shared/util';
 import { type RuleConditionEntity } from 'loot-core/types/models/rule';
 
-import { useCategories } from '../../../hooks/useCategories';
 import { useFilters } from '../../../hooks/useFilters';
 import { useLocalPref } from '../../../hooks/useLocalPref';
 import { useNavigate } from '../../../hooks/useNavigate';
@@ -30,8 +29,6 @@ import { createSpendingSpreadsheet } from '../spreadsheets/spending-spreadsheet'
 import { useReport } from '../useReport';
 
 export function Spending() {
-  const categories = useCategories();
-
   const {
     conditions,
     conditionsOp,
@@ -71,13 +68,12 @@ export function Spending() {
   const getGraphData = useMemo(() => {
     setDataCheck(false);
     return createSpendingSpreadsheet({
-      categories,
       conditions,
       conditionsOp,
       setDataCheck,
       compare,
     });
-  }, [categories, conditions, conditionsOp, compare]);
+  }, [conditions, conditionsOp, compare]);
 
   const data = useReport('default', getGraphData);
   const navigate = useNavigate();

packages/desktop-client/src/components/reports/reports/SpendingCard.tsx

@@ -3,7 +3,6 @@ import React, { useState, useMemo } from 'react';
 import * as monthUtils from 'loot-core/src/shared/months';
 import { amountToCurrency } from 'loot-core/src/shared/util';
 
-import { useCategories } from '../../../hooks/useCategories';
 import { useLocalPref } from '../../../hooks/useLocalPref';
 import { styles } from '../../../style/styles';
 import { theme } from '../../../style/theme';
@@ -18,8 +17,6 @@ import { createSpendingSpreadsheet } from '../spreadsheets/spending-spreadsheet'
 import { useReport } from '../useReport';
 
 export function SpendingCard() {
-  const categories = useCategories();
-
   const [isCardHovered, setIsCardHovered] = useState(false);
   const [spendingReportFilter = ''] = useLocalPref('spendingReportFilter');
   const [spendingReportTime = 'lastMonth'] = useLocalPref('spendingReportTime');
@@ -30,12 +27,11 @@ export function SpendingCard() {
   const parseFilter = spendingReportFilter && JSON.parse(spendingReportFilter);
   const getGraphData = useMemo(() => {
     return createSpendingSpreadsheet({
-      categories,
       conditions: parseFilter.conditions,
       conditionsOp: parseFilter.conditionsOp,
       compare: spendingReportCompare,
     });
-  }, [categories, parseFilter, spendingReportCompare]);
+  }, [parseFilter, spendingReportCompare]);
 
   const data = useReport('default', getGraphData);
   const todayDay =

packages/desktop-client/src/components/reports/spreadsheets/custom-spreadsheet.ts

@@ -39,7 +39,6 @@ export type createCustomSpreadsheetProps = {
   endDate: string;
   interval: string;
   categories: { list: CategoryEntity[]; grouped: CategoryGroupEntity[] };
-  selectedCategories: CategoryEntity[];
   conditions: RuleConditionEntity[];
   conditionsOp: string;
   showEmpty: boolean;
@@ -60,7 +59,6 @@ export function createCustomSpreadsheet({
   endDate,
   interval,
   categories,
-  selectedCategories,
   conditions = [],
   conditionsOp,
   showEmpty,
@@ -77,14 +75,6 @@ export function createCustomSpreadsheet({
 }: createCustomSpreadsheetProps) {
   const [categoryList, categoryGroup] = categoryLists(categories);
 
-  const categoryFilter = (categories.list || []).filter(
-    category =>
-      selectedCategories &&
-      selectedCategories.some(
-        selectedCategory => selectedCategory.id === category.id,
-      ),
-  );
-
   const [groupByList, groupByLabel]: [
     groupByList: UncategorizedEntity[],
     groupByLabel: 'category' | 'categoryGroup' | 'payee' | 'account',
@@ -112,7 +102,6 @@ export function createCustomSpreadsheet({
           startDate,
           endDate,
           interval,
-          categoryFilter,
           conditionsOpKey,
           filters,
         ),
@@ -123,7 +112,6 @@ export function createCustomSpreadsheet({
           startDate,
           endDate,
           interval,
-          categoryFilter,
           conditionsOpKey,
           filters,
         ),

packages/desktop-client/src/components/reports/spreadsheets/grouped-spreadsheet.ts

@@ -25,7 +25,6 @@ export function createGroupedSpreadsheet({
   endDate,
   interval,
   categories,
-  selectedCategories,
   conditions = [],
   conditionsOp,
   showEmpty,
@@ -37,14 +36,6 @@ export function createGroupedSpreadsheet({
 }: createCustomSpreadsheetProps) {
   const [categoryList, categoryGroup] = categoryLists(categories);
 
-  const categoryFilter = (categories.list || []).filter(
-    category =>
-      selectedCategories &&
-      selectedCategories.some(
-        selectedCategory => selectedCategory.id === category.id,
-      ),
-  );
-
   return async (
     spreadsheet: ReturnType<typeof useSpreadsheet>,
     setData: (data: GroupedEntity[]) => void,
@@ -67,7 +58,6 @@ export function createGroupedSpreadsheet({
           startDate,
           endDate,
           interval,
-          categoryFilter,
           conditionsOpKey,
           filters,
         ),
@@ -78,7 +68,6 @@ export function createGroupedSpreadsheet({
           startDate,
           endDate,
           interval,
-          categoryFilter,
           conditionsOpKey,
           filters,
         ),

packages/desktop-client/src/components/reports/spreadsheets/makeQuery.ts

@@ -1,5 +1,4 @@
 import { q } from 'loot-core/src/shared/query';
-import { type CategoryEntity } from 'loot-core/src/types/models';
 
 import { ReportOptions } from '../ReportOptions';
 
@@ -8,7 +7,6 @@ export function makeQuery(
   startDate: string,
   endDate: string,
   interval: string,
-  categoryFilter: CategoryEntity[],
   conditionsOpKey: string,
   filters: unknown[],
 ) {
@@ -24,19 +22,6 @@ export function makeQuery(
       : '$' + ReportOptions.intervalMap.get(interval)?.toLowerCase() || 'month';
 
   const query = q('transactions')
-    //Apply Category_Selector
-    .filter(
-      categoryFilter && {
-        $or: [
-          {
-            category: null,
-            $or: categoryFilter.map(category => ({
-              category: category.id,
-            })),
-          },
-        ],
-      },
-    )
     //Apply filters and split by "Group By"
     .filter({
       [conditionsOpKey]: filters,

packages/desktop-client/src/components/reports/spreadsheets/spending-spreadsheet.ts

@@ -6,11 +6,7 @@ import { type useSpreadsheet } from 'loot-core/src/client/SpreadsheetProvider';
 import { send } from 'loot-core/src/platform/client/fetch';
 import * as monthUtils from 'loot-core/src/shared/months';
 import { integerToAmount } from 'loot-core/src/shared/util';
-import {
-  type CategoryEntity,
-  type RuleConditionEntity,
-  type CategoryGroupEntity,
-} from 'loot-core/src/types/models';
+import { type RuleConditionEntity } from 'loot-core/src/types/models';
 import {
   type SpendingMonthEntity,
   type SpendingEntity,
@@ -21,7 +17,6 @@ import { getSpecificRange } from '../reportRanges';
 import { makeQuery } from './makeQuery';
 
 type createSpendingSpreadsheetProps = {
-  categories: { list: CategoryEntity[]; grouped: CategoryGroupEntity[] };
   conditions?: RuleConditionEntity[];
   conditionsOp?: string;
   setDataCheck?: (value: boolean) => void;
@@ -29,7 +24,6 @@ type createSpendingSpreadsheetProps = {
 };
 
 export function createSpendingSpreadsheet({
-  categories,
   conditions = [],
   conditionsOp,
   setDataCheck,
@@ -67,7 +61,6 @@ export function createSpendingSpreadsheet({
           lastYearStartDate,
           endDate,
           interval,
-          categories.list,
           conditionsOpKey,
           filters,
         ),
@@ -78,7 +71,6 @@ export function createSpendingSpreadsheet({
           lastYearStartDate,
           endDate,
           interval,
-          categories.list,
           conditionsOpKey,
           filters,
         ),

packages/loot-core/migrations/1722717601000_reports_move_selected_categories.js

@@ -0,0 +1,55 @@
+export default async function runMigration(db) {
+  const categories = await db.runQuery(
+    'SELECT id FROM categories WHERE tombstone = 0',
+    [],
+    true,
+  );
+
+  const customReports = await db.runQuery(
+    'SELECT id, selected_categories, conditions FROM custom_reports WHERE tombstone = 0 AND selected_categories IS NOT NULL',
+    [],
+    true,
+  );
+
+  // Move all `selected_categories` to `conditions` if possible.. otherwise skip
+  for (const report of customReports) {
+    const conditions = report.conditions ? JSON.parse(report.conditions) : [];
+    const selectedCategories = report.selected_categories
+      ? JSON.parse(report.selected_categories)
+      : [];
+    const selectedCategoryIds = selectedCategories.map(({ id }) => id);
+
+    const areAllCategoriesSelected = !categories.find(
+      ({ id }) => !selectedCategoryIds.includes(id),
+    );
+
+    // Do nothing if all categories are selected.. we don't need to add a new condition for that
+    if (areAllCategoriesSelected) {
+      continue;
+    }
+
+    // If `conditions` already has a "category" filter - skip the entry
+    if (conditions.find(({ field }) => field === 'category')) {
+      continue;
+    }
+
+    // Append a new condition with the selected category IDs
+    await db.runQuery('UPDATE custom_reports SET conditions = ? WHERE id = ?', [
+      JSON.stringify([
+        ...conditions,
+        {
+          field: 'category',
+          op: 'oneOf',
+          value: selectedCategoryIds,
+          type: 'id',
+        },
+      ]),
+      report.id,
+    ]);
+  }
+
+  // Remove all the `selectedCategories` values - we don't need them anymore
+  await db.runQuery(
+    'UPDATE custom_reports SET selected_categories = NULL WHERE tombstone = 0',
+  );
+}

packages/loot-core/src/client/data-hooks/reports.ts

@@ -25,7 +25,6 @@ function toJS(rows: CustomReportData[]) {
       showHiddenCategories: row.show_hidden === 1,
       includeCurrentInterval: row.include_current === 1,
       showUncategorized: row.show_uncategorized === 1,
-      selectedCategories: row.selected_categories,
       graphType: row.graph_type,
       conditions: row.conditions,
       conditionsOp: row.conditions_op ?? 'and',

packages/loot-core/src/server/aql/schema/index.ts

@@ -144,7 +144,6 @@ export const schema = {
     show_hidden: f('integer', { default: 0 }),
     show_uncategorized: f('integer', { default: 0 }),
     include_current: f('integer', { default: 0 }),
-    selected_categories: f('json'),
     graph_type: f('string', { default: 'BarGraph' }),
     conditions: f('json'),
     conditions_op: f('string'),

packages/loot-core/src/server/migrate/migrations.ts

@@ -6,6 +6,7 @@ import { Database } from '@jlongster/sql.js';
 import { v4 as uuidv4 } from 'uuid';
 
 import m1632571489012 from '../../../migrations/1632571489012_remove_cache';
+import m1722717601000 from '../../../migrations/1722717601000_reports_move_selected_categories';
 import * as fs from '../../platform/server/fs';
 import * as sqlite from '../../platform/server/sqlite';
 
@@ -13,6 +14,7 @@ let MIGRATIONS_DIR = fs.migrationsPath;
 
 const javascriptMigrations = {
   1632571489012: m1632571489012,
+  1722717601000: m1722717601000,
 };
 
 export async function withMigrationsDir(

packages/loot-core/src/server/reports/app.ts

@@ -42,7 +42,6 @@ const reportModel = {
       showHiddenCategories: row.show_hidden === 1,
       showUncategorized: row.show_uncategorized === 1,
       includeCurrentInterval: row.include_current === 1,
-      selectedCategories: row.selected_categories,
       graphType: row.graph_type,
       conditions: row.conditions,
       conditionsOp: row.conditions_op,
@@ -66,7 +65,6 @@ const reportModel = {
       show_hidden: report.showHiddenCategories ? 1 : 0,
       show_uncategorized: report.showUncategorized ? 1 : 0,
       include_current: report.includeCurrentInterval ? 1 : 0,
-      selected_categories: report.selectedCategories,
       graph_type: report.graphType,
       conditions: report.conditions,
       conditions_op: report.conditionsOp,

packages/loot-core/src/shared/rules.ts

@@ -273,6 +273,12 @@ export function makeValue(value, cond) {
     default:
   }
 
+  const isMulti = ['oneOf', 'notOneOf'].includes(cond.op);
+
+  if (isMulti) {
+    return { ...cond, error: null, value: value || [] };
+  }
+
   return { ...cond, error: null, value };
 }
 

packages/loot-core/src/types/models/reports.d.ts

@@ -1,4 +1,3 @@
-import { CategoryEntity } from './category';
 import { type RuleConditionEntity } from './rule';
 
 export interface CustomReportEntity {
@@ -17,7 +16,6 @@ export interface CustomReportEntity {
   showHiddenCategories: boolean;
   includeCurrentInterval: boolean;
   showUncategorized: boolean;
-  selectedCategories?: CategoryEntity[];
   graphType: string;
   conditions?: RuleConditionEntity[];
   conditionsOp: 'and' | 'or';
@@ -140,7 +138,6 @@ export interface CustomReportData {
   show_hidden: number;
   include_current: number;
   show_uncategorized: number;
-  selected_categories?: CategoryEntity[];
   graph_type: string;
   conditions?: RuleConditionEntity[];
   conditions_op: 'and' | 'or';

upcoming-release-notes/3178.md

@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [MatissJanis]
+---
+
+Custom reports: unify `selectedCategories` and `conditions` data source.