import { DecimalPipe } from '@angular/common';
import { HttpErrorResponse } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { ApplicationFileService } from '@core/services/application-file.service';
import { CurrencyService } from '@core/services/currency.service';
import { FileUploadProgressService } from '@core/services/file-upload-progress.service';
import { TimeZoneService } from '@core/services/time-zone.service';
import { TranslationService } from '@core/services/translation.service';
import { AdHocReportingAPI } from '@core/typings/api/ad-hoc-reporting.typing';
import { ReferenceFieldAPI } from '@core/typings/api/reference-fields.typing';
import { BaseApplication } from '@core/typings/application.typing';
import { AdHocReportingUI } from '@core/typings/ui/ad-hoc-reporting.typing';
import { Automation } from '@core/typings/ui/automation.typing';
import { ReferenceFieldsUI, STANDARD_FIELDS_CATEGORY_ID } from '@core/typings/ui/reference-fields.typing';
import { ClientSettingsService } from '@features/client-settings/client-settings.service';
import { FormAudience, FormData, FormDecisionTypes, FormDefinitionComponent, FormDefinitionForUi, FormioAnswerValues, FormioChanges, FormioChangeTracker } from '@features/configure-forms/form.typing';
import { CustomDataTablesService } from '@features/custom-data-tables/custom-data-table.service';
import { CustomDataTable, PicklistDataType } from '@features/custom-data-tables/custom-data-tables.typing';
import { DefaultValType } from '@features/formio/component-configuration/component-configuration.typing';
import { SpecialHandling } from '@features/formio/formio-components/standard-formio-components/gc-special-handling/gc-special-handling.component';
import { ComponentHelperService } from '@features/formio/services/component-helper/component-helper.service';
import { InKindRequestedItem } from '@features/in-kind/in-kind.typing';
import { UserService } from '@features/users/user.service';
import { ALL_SKIP_FILTER, ArrayHelpersService, AutoTableRepositoryFactory, Base64, CSVBoolean, CSVDate, DynamicCSVImportService, FileService, InflectService, IsNumber, IsOneOf, IsString, PaginationOptions, Required, RequiredFormCheckbox, SimpleNumberMap, SimpleStringMap, TableDataDownloadFormat, TopLevelFilter, TopLevelFilterOption, TypeaheadSelectOption, YcFile } from '@yourcause/common';
import { I18nService } from '@yourcause/common/i18n';
import { LogService } from '@yourcause/common/logging';
import { CurrencyValue } from '@yourcause/common/masking';
import { NotifierService } from '@yourcause/common/notifier';
import { AttachYCState, BaseYCService } from '@yourcause/common/state';
import { intersection, uniq } from 'lodash';
import moment from 'moment';
import { ReferenceFieldsResources } from '../reference-fields.resources';
import { ReferenceFieldsState } from '../reference-fields.state';

export const STANDARD_FIELD_PREFIX = 'gcStandardField_';
export const REF_COMPONENT_TYPE_PREFIX = 'referenceFields-';

@Injectable({ providedIn: 'root' })
@AttachYCState(ReferenceFieldsState)
export class ReferenceFieldsService extends BaseYCService<ReferenceFieldsState> {
  private decimal = new DecimalPipe('en-US');
  referenceFieldTypeList = ReferenceFieldsUI.ReferenceFieldTypeList;

   constructor (
    private fileUploadProgressService: FileUploadProgressService,
    private logger: LogService,
    private applicationFileService: ApplicationFileService,
    private i18n: I18nService,
    private referenceFieldsResources: ReferenceFieldsResources,
    private inflect: InflectService,
    private customDataTablesService: CustomDataTablesService,
    private userService: UserService,
    private notifier: NotifierService,
    private fileService: FileService,
    private arrayHelper: ArrayHelpersService,
    private translationService: TranslationService,
    private autoTableFactory: AutoTableRepositoryFactory,
    private timezoneService: TimeZoneService,
    private clientSettingsService: ClientSettingsService,
    private currencyService: CurrencyService,
    private dynamicCsvService: DynamicCSVImportService,
    private componentHelper: ComponentHelperService
  ) {
    super();
  }

  get decisionOptions () {
    return [{
      label: this.i18n.translate(
        'common:textYes',
        {},
        'Yes'
      ),
      value: FormDecisionTypes.Approve
    }, {
      label: this.i18n.translate(
        'common:textNo',
        {},
        'No'
      ),
      value: FormDecisionTypes.Decline
    }, {
      label: this.i18n.translate(
        'common:textRecused',
        {},
        'Recused'
      ),
      value: FormDecisionTypes.Recused
    }];
  }
  get dataTables () {
    return this.customDataTablesService.customDataTableOptionsMap;
  }

  get allReferenceFields () {
    return this.get('allReferenceFields');
  }

  get referenceFieldMap () {
    return this.get('referenceFieldMap');
  }

  get referenceFieldMapById () {
    return this.get('referenceFieldMapById');
  }

  get isParentRefFieldMap () {
    return this.get('isParentRefFieldMap');
  }

  get tableColumnsMap () {
    return this.get('tableColumnsMap');
  }

  get dataPointsMap () {
    return this.get('dataPointsMap');
  }

  get applicationFormTableRowsMap () {
    return this.get('applicationFormTableRowsMap');
  }

  get categories () {
    return this.get('categories');
  }

  get categoryNameMap () {
    return this.get('categoryNameMap');
  }

  get parentPicklistValueMap () {
    return this.get('parentPicklistValueMap');
  }

  get categoryOptions () {
    return this.get('categoryOptions');
  }

  get categoryTokenGroups () {
    return this.get('categoryTokenGroups');
  }

  get currentFormRefFields () {
    return this.get('currentFormRefFields');
  }

  get allTypes () {
    return this.get('allTypes');
  }

  get formFieldMask () {
    return '******';
  }

  getIdNameMap () {
    return this.allReferenceFields.reduce((acc, ref) => {
      return {
        ...acc,
        [ref.referenceFieldId]: ref.name
      };
    }, {});
  }

  getFieldOption (
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ): TypeaheadSelectOption<ReferenceFieldsUI.ReferenceFieldTypes> {
    return {
      label: this.getFieldTypeTranslatedWithIcon(type).label,
      value: type
    };
  }

  getFieldTypeTranslatedWithIcon (
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ): { label: string; icon: string } {
    switch (type) {
      case ReferenceFieldsUI.ReferenceFieldTypes.Subset:
        return {
          label: this.i18n.translate(
            'common:textFieldGroup',
            {},
            'Field group'
          ),
          icon: 'list'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Currency:
        return {
          label: this.i18n.translate(
            'common:textCurrencyField',
            {},
            'Currency field'
          ),
          icon: 'money-bill'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Aggregate:
        return {
          label: this.i18n.translate(
            'common:textAggregationField',
            {},
            'Aggregation field'
          ),
          icon: 'plus'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Checkbox:
        return {
          label: this.i18n.translate(
            'common:textCheckbox',
            {},
            'Checkbox'
          ),
          icon: 'check'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
        return {
          label: this.i18n.translate(
            'common:textPicklist',
            {},
            'Picklist'
          ),
          icon: 'list-alt'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Date:
        return {
          label: this.i18n.translate(
            'common:hdrDate',
            {},
            'Date'
          ),
          icon: 'calendar'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
        return {
          label: this.i18n.translate(
            'common:textFileUpload',
            {},
            'File upload'
          ),
          icon: 'upload'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Number:
        return {
          label: this.i18n.translate(
            'common:textNumber',
            {},
            'Number'
          ),
          icon: 'hashtag'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
        return {
            label: this.i18n.translate(
            'common:textRadioButtons',
            {},
            'Radio buttons'
          ),
          icon: 'dot-circle'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes:
        return {
          label: this.i18n.translate(
            'common:textSelectBoxes',
            {},
            'Select boxes'
          ),
          icon: 'ballot-check'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.TextArea:
        return {
          label: this.i18n.translate(
            'common:textTextArea',
            {},
            'Text area'
          ),
          icon: 'align-left'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.TextField:
        return {
          label: this.i18n.translate(
            'common:textText',
            {},
            'Text'
          ),
          icon: 'font'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.ExternalAPI:
        return {
          label: this.i18n.translate(
            'common:textExternalAPI',
            {},
            'External API'
          ),
          icon: 'arrow-up-right-from-square'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Table:
        return {
          label: this.i18n.translate(
            'FORMS:textTable',
            {},
            'Table'
          ),
          icon: 'table'
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.DataPoint:
        return {
          label: this.i18n.translate(
            'FORMS:textFieldGroupOption',
            {},
            'Field group option'
          ),
          icon: 'triangle'
        };
    }
  }

  getFormattingTypeOptions (): TypeaheadSelectOption<ReferenceFieldAPI.ReferenceFieldFormattingType>[] {
    const baseList = this.arrayHelper.sort([{
      label: this.i18n.translate(
        'common:textEmailWithExample',
        {},
        'Email (e.g. name@host.com)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.EMAIL
    }, {
      label: this.i18n.translate(
        'common:textTimeTwelveHourWithExample',
        {},
        'Time (12 hour, e.g. 12:34 PM)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.TIME_12_HOUR
    }, {
      label: this.i18n.translate(
        'common:textTimeTwentyFourHourWithExample',
        {},
        'Time (24 hour, e.g. 21:00)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.TIME_24_HOUR
    }, {
      label: this.i18n.translate(
        'common:textEINWithExample',
        {},
        'EIN (e.g. 12-1234567)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.EIN
    }, {
      label: this.i18n.translate(
        'common:textURLWithExample',
        {},
        'URL (e.g. www.website.com)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.URL_OPT_HTTP
    }, {
      label: this.i18n.translate(
        'common:textURLHttpRequiredWithExample',
        {},
        'URL (http/https required, e.g. http://website.com)'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.URL_REQ_HTTP
    }]);

    baseList.unshift({
      label: this.i18n.translate(
        'common:lblNoFormatting',
        {},
        'No formatting'
      ),
      value: ReferenceFieldAPI.ReferenceFieldFormattingType.NONE
    });

    return baseList;
  }

  getSubsetCollectionOptions (): TypeaheadSelectOption<ReferenceFieldAPI.DataSetCollectionType>[] {
    return this.arrayHelper.sort([{
      label: this.i18n.translate('common:textNumber', {}, 'Number'),
      value: ReferenceFieldAPI.DataSetCollectionType.Number
    }, {
      label: this.i18n.translate('common:textPercentage', {}, 'Percentage'),
      value: ReferenceFieldAPI.DataSetCollectionType.Percent
    }, {
      label: this.i18n.translate('common:textYesOrNo', {}, 'Yes/No'),
      value: ReferenceFieldAPI.DataSetCollectionType.YesOrNo
    }]);
  }

  getAggregateTypeOptions (): TypeaheadSelectOption<ReferenceFieldAPI.ReferenceFieldAggregateType>[] {
    return this.arrayHelper.sort([{
      label: this.i18n.translate('GLOBAL:textSum', {}, 'Sum'),
      value: ReferenceFieldAPI.ReferenceFieldAggregateType.Sum
    }, {
      label: this.i18n.translate('GLOBAL:textMax', {}, 'Max'),
      value: ReferenceFieldAPI.ReferenceFieldAggregateType.Max
    }, {
      label: this.i18n.translate('GLOBAL:textMin', {}, 'Min'),
      value: ReferenceFieldAPI.ReferenceFieldAggregateType.Min
    }, {
      label: this.i18n.translate('GLOBAL:textCount', {}, 'Count'),
      value: ReferenceFieldAPI.ReferenceFieldAggregateType.Count
    }, {
      label: this.i18n.translate('GLOBAL:textAverage', {}, 'Average'),
      value: ReferenceFieldAPI.ReferenceFieldAggregateType.Average
    }]);
  }

  getReferenceFieldTypeOptions (): TypeaheadSelectOption<ReferenceFieldsUI.ReferenceFieldTypes>[] {
    const types = this.referenceFieldTypeList;

    return this.arrayHelper.sort(
      types.map((type) => {
        return this.getFieldOption(type);
      }),
      'label'
    );
  }

  getTypeSelectOptionsForFilters (
    hideOptionForAll = false,
    includeStandardProductFieldOption = false
  ) {
    const typeOptions: TopLevelFilterOption[] = [
      ...this.getReferenceFieldTypeOptions().map((opt) => {
        let label = opt.label;
        const value = opt.value;
        let customColumn: string;
        let customValue: string;
        let filterTypeOverride: string;
        if (opt.value === ReferenceFieldsUI.ReferenceFieldTypes.Aggregate) {
          label = this.i18n.translate(
            'common:textAggregationField',
            {},
            'Aggregation field'
          );
          customColumn = 'aggregateType';
          customValue = '';
          filterTypeOverride = 'nb';
        }

        return {
          label,
          value,
          customColumn,
          customValue,
          filterTypeOverride
        };
      }),
      includeStandardProductFieldOption ?
        {
          label: this.i18n.translate(
            'common:textStandardProductFields',
            {},
            'Standard product fields'
          ),
          value: true,
          customColumn: 'isStandardProductField'
        } :
        undefined
    ].filter((item) => !!item);
    const options = this.arrayHelper.sort(typeOptions, 'label');

    return [
      !hideOptionForAll ?
        {
          display: this.i18n.translate(
            'GLOBAL:textAllFormFieldTypes',
            {},
            'All form field types'
          ),
          value: ALL_SKIP_FILTER
        } : undefined,
      ...options
    ].filter((item) => !!item);
  }

  getCollectionTypeMap (): Record<ReferenceFieldAPI.DataSetCollectionType, string> {
    const options = this.getSubsetCollectionOptions();

    return options.reduce((acc, option) => {
      return {
        ...acc,
        [option.value]: option.label
      };
    }, {} as Record<ReferenceFieldAPI.DataSetCollectionType, string>);
  }

  getReferenceFieldTypeToLabelMap () {
    const map = {} as Record<ReferenceFieldsUI.ReferenceFieldTypes, string>;
    const options = this.getReferenceFieldTypeOptions();

    return options.reduce((acc, option) => {
      return {
        ...acc,
        [option.value]: option.label
      };
    }, map);
  }

  getFormattingMap () {
    const options = this.getFormattingTypeOptions();

    return options.reduce<Record<ReferenceFieldAPI.ReferenceFieldFormattingType, string>>((acc, option) => {
      return {
        ...acc,
        [option.value]: option.label
      };
    }, {} as Record<ReferenceFieldAPI.ReferenceFieldFormattingType, string>);
  }

  findReferenceField (key: string) {
    return this.allReferenceFields.find((field) => {
      return field.key === key;
    });
  }

  findReferenceFieldById (id: number) {
    return this.allReferenceFields.find((field) => {
      return field.referenceFieldId === id;
    });
  }

  async resolve () {
    await Promise.all([
      this.fetchAllCategories(),
      this.fetchAllFields()
    ]);
  }

  async resetFieldsAndCategories () {
    await Promise.all([
      this.resetFields(),
      this.resetCategories()
    ]);
  }

  resetRefFieldRepo () {
    const repo = this.autoTableFactory.getRepository('REFERENCE_FIELDS');
    if (repo) {
      repo.reset();
    }
  }

  resetCurrentFormRefFields () {
    this.set('currentFormRefFields', undefined);
  }

  resetTableColumnsMap (tableId: number) {
    this.set('tableColumnsMap', {
      ...this.tableColumnsMap,
      [tableId]: undefined
    });
  }

  resetDataPointsMap (subsetId: number) {
    this.set('dataPointsMap', {
      ...this.dataPointsMap,
      [subsetId]: undefined
    });
  }

  getIsParentRefField (refFieldId: number) {
    return this.allReferenceFields.some((field) => {
      return field.parentReferenceFieldId === refFieldId;
    });
  }

  getChildOfParentRefField (refFieldId: number) {
    return this.allReferenceFields.find((field) => {
      return field.parentReferenceFieldId === refFieldId;
    });
  }

  setParentPicklistValueMap (
    refFieldId: number,
    value: string|string[]
  ) {
    const isParent = this.getIsParentRefField(refFieldId);
    if (isParent) {
      this.set('parentPicklistValueMap', {
        ...this.parentPicklistValueMap,
        [refFieldId]: value
      });
    }
  }

  resetParentPicklistValueMap () {
    this.set('parentPicklistValueMap', {});
  }

  async resetFields () {
    this.set('allReferenceFields', undefined);
    await this.fetchAllFields();
    this.resetRefFieldRepo();
  }

  async resetCategories () {
    this.set('categories', undefined);
    await this.fetchAllCategories();
  }

  getReferenceFieldFromCompType (type: string) {
    const key = this.componentHelper.getRefFieldKeyFromCompType(type);

    return this.getReferenceFieldByKey(key);
  }

  async getCdtOptionsFromRefField (
    fieldId: number
  ): Promise<TypeaheadSelectOption[]> {
    const field = this.referenceFieldMapById[fieldId];
    let options: TypeaheadSelectOption[] = [];
    if (field.customDataTableGuid) {
      if (!this.dataTables[field.customDataTableGuid]) {
        await this.customDataTablesService.setCustomDataTableOptionsFromGuid(
          field.customDataTableGuid,
          true,
          this.userService.getCurrentUserCulture()
        );
      }

      options = this.dataTables[field.customDataTableGuid].map((option) => {
        return {
          label: option.value,
          value: option.key
        };
      });
    }

    return options;
  }

  getReferenceFieldColumnDef (
    field: ReferenceFieldAPI.ReferenceFieldAdHocModel,
    isClientSide = false
  ): AdHocReportingUI.ColumnDefinition[] {
    let base: AdHocReportingUI.ColumnDefinition = {
      i18nKey: '',
      display: field.name,
      column: field.key,
      type: 'text',
      isReferenceField: true,
      isStandardField: field.isStandardProductField,
      formIds: field.formIds,
      noFiltering: field.isEncrypted || field.isMasked
    };

    if (field.customDataTableGuid) {
      const filterOptions = (this.dataTables[field.customDataTableGuid] || [])
        .map((opt) => {
          return {
            label: opt.value,
            value: opt.key
          };
        });
      base = {
        ...base,
        type: field.supportsMultiple ?
          'multiValueList' :
          'multiListFuzzyText',
        filterOptions,
        format: 'label'
      } as AdHocReportingUI.TypeaheadSelectColumnDefinition;
    }

    switch (field.type) {
      default:
      case ReferenceFieldsUI.ReferenceFieldTypes.ExternalAPI:
      case ReferenceFieldsUI.ReferenceFieldTypes.TextField:
      case ReferenceFieldsUI.ReferenceFieldTypes.TextArea:
        return [{
          ...base,
          type: field.supportsMultiple ?
            'multiValueText' :
            base.type
        }] as AdHocReportingUI.DefaultColumnDefinition[];
      case ReferenceFieldsUI.ReferenceFieldTypes.Currency:
        const currencyText = this.i18n.translate('common:textCurrency', {}, 'Currency');

        const currencyValueKey = field.key + '.currencyValue';
        const currencyColumn: AdHocReportingUI.ColumnDefinition = {
          i18nKey: '',
          supportsGrouping: false,
          display: field.name + ' ' + currencyText,
          column: currencyValueKey,
          type: 'text',
          isReferenceField: true,
          isStandardField: field.isStandardProductField,
          formIds: field.formIds,
          noFiltering: field.isEncrypted || field.isMasked
        };

        return [
          {
            ...base,
            type: 'number',
            format: 'decimal'
          },
          {
            ...currencyColumn
          }
        ] as AdHocReportingUI.NumberColumnDefinition[];
      case ReferenceFieldsUI.ReferenceFieldTypes.DataPoint:
        const relatedField = this.allReferenceFields.find((refField) => {
          return refField.referenceFieldTableId === field.referenceFieldTableId;
        });
        const label = `${field.name} (${relatedField.name})`;

        switch (relatedField.subsetCollectionType) {
          default:
          case ReferenceFieldAPI.DataSetCollectionType.Number:
            return [{
              ...base,
              display: label,
              referenceFieldTableId: field.referenceFieldTableId,
              type: 'number',
              format: 'number',
              parentTableDisplay: relatedField.name
            }] as AdHocReportingUI.NumberColumnDefinition[];
          case ReferenceFieldAPI.DataSetCollectionType.Percent:
            return [{
              ...base,
              display: label,
              referenceFieldTableId: field.referenceFieldTableId,
              type: 'number',
              format: 'wholeNumberPercent',
              parentTableDisplay: relatedField.name
            }] as AdHocReportingUI.NumberColumnDefinition[];
          case ReferenceFieldAPI.DataSetCollectionType.YesOrNo:
            return [{
              ...base,
              display: label,
              referenceFieldTableId: field.referenceFieldTableId,
              type: 'boolean',
              parentTableDisplay: relatedField.name
            }] as AdHocReportingUI.DefaultColumnDefinition[];
        }
      case ReferenceFieldsUI.ReferenceFieldTypes.Number:
      case ReferenceFieldsUI.ReferenceFieldTypes.Aggregate:
        return [{
          ...base,
          type: 'number',
          format: 'decimal'
        }] as AdHocReportingUI.NumberColumnDefinition[];
      case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
      case ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes:
      case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
        return [{
          ...base,
          supportsGrouping: !field.supportsMultiple
        }];
      case ReferenceFieldsUI.ReferenceFieldTypes.Date:
        return [{
          ...base,
          type: 'date',
          format: 'date'
        }] as AdHocReportingUI.DateColumnDefinition[];
      case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
        return [{
          ...base,
          format: 'file'
        }] as AdHocReportingUI.FileColumnDefinition[];
      case ReferenceFieldsUI.ReferenceFieldTypes.Checkbox:
        return [{
          ...base,
          type: 'list',
          format: 'label',
          filterOptions: [{
            label: this.i18n.translate(
              'common:textIsTrue',
              {},
              'Is true'
            ),
            value: isClientSide ? true : 'true'
          }, {
            label: this.i18n.translate(
              'common:textIsFalse',
              {},
              'Is false'
            ),
            value: isClientSide ? false : 'false'
          }]
        }];
    }
  }

  async adaptFieldIdsForAutomation (
    referenceFieldIds: number[]
  ): Promise<Automation.ObjectConfig> {
    if (referenceFieldIds.length === 0) {
      return undefined;
    }
    const guids = uniq(referenceFieldIds.map((id) => {
      const field = this.findReferenceFieldById(id);

      return field.customDataTableGuid;
    }).filter((guid) => !!guid));
    for (const guid of guids) {
      await this.customDataTablesService.setCustomDataTableOptionsFromGuid(
        guid,
        true,
        this.userService.getCurrentUserCulture()
      );
    }

    return {
      label: this.i18n.translate(
        'GLOBAL:textGrantManagerFields',
        {},
        'Grant manager fields'
      ),
      key: 'referenceFields',
      columns: await Promise.all(referenceFieldIds.map((id) => {
        const field = this.findReferenceFieldById(id);

        return this.refFieldTypeToAutomation(
          field.name,
          field
        );
      }))
    };
  }

  formRefFieldToAutomationType (
    formRefField: FormDefinitionComponent
  ): Automation.ObjectColumnConfig {
    const refFieldKey = formRefField.type.split('-')[1];
    const refField = this.getReferenceFieldByKey(refFieldKey);

    return this.refFieldTypeToAutomation(
      formRefField.label,
      refField
    );
  }

  refFieldTypeToAutomation (
    label: string,
    refField: ReferenceFieldAPI.ReferenceFieldDisplayModel
  ): Automation.ObjectColumnConfig {
    const baseConfig: Automation.CoreObjectColumnType = {
      label,
      key: refField.key
    };

    switch (refField.type) {
      case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
      case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
      case ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes:
        const selectedCustomDataTable = refField.customDataTableGuid;
        const options = this.dataTables[selectedCustomDataTable];

        return <Automation.ObjectColumnSelectType>{
          ...baseConfig,
          comparison: Automation.Comparisons.SelectIs,
          options: options ? options.map((item) => {
            return {
              label: item.value,
              value: item.key
            };
          }) : []
        };

      case ReferenceFieldsUI.ReferenceFieldTypes.Number:
      case ReferenceFieldsUI.ReferenceFieldTypes.Aggregate:
        return {
          ...baseConfig,
          comparison: Automation.Comparisons.NumberRange
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Date:
        return {
          ...baseConfig,
          comparison: Automation.Comparisons.DateRange
        };
      default:
      case ReferenceFieldsUI.ReferenceFieldTypes.TextArea:
      case ReferenceFieldsUI.ReferenceFieldTypes.ExternalAPI:
      case ReferenceFieldsUI.ReferenceFieldTypes.TextField:
        return {
          ...baseConfig,
          comparison: Automation.Comparisons.TextIs
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Checkbox:
        return {
          ...baseConfig,
          comparison: Automation.Comparisons.IsSelected
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
        return {
          ...baseConfig,
          comparison: Automation.Comparisons.IsUploaded
        };
    }
  }

  getReferenceFieldByKey (key: string) {
    return this.referenceFieldMap[key];
  }

  async getReferenceFieldResponses (
    appId: number,
    applicationFormId: number,
    formDefinition: FormDefinitionForUi[], /* used to filter out field responses that no longer exist on this form */
    tableAndSubsetIds: number[],
    revisionId: number, /* Pass revisionId if appFormId not created yet (Offline) */
    skipAddTablesToState = false /* This is only necessary when editing a form */,
    onlyFetchTableResponses = false /* For after import of table rows */,
    fetchedResponses?: ReferenceFieldAPI.ApplicationRefFieldResponse[],
    formData?: FormData // only used for eligibility forms. We need to add these answers to reference fields responses so it populates
  ): Promise<ReferenceFieldsUI.RefResponseMap> {
    const tableResponses = await this.getTableResponses(
      appId,
      applicationFormId,
      tableAndSubsetIds
    );
    let mapped: ReferenceFieldsUI.RefResponseMapForAdapting = {};
    if (!onlyFetchTableResponses) {
      let responses = fetchedResponses;
      if (!responses) {
        responses = await this.referenceFieldsResources.getReferenceFieldResponses(
          appId,
          applicationFormId
        );
      }
      if (formData) {
        const map = this.componentHelper.findFormDataToAddToResponses(formData, formDefinition);
        Object.keys(map).forEach((referenceFieldKey) => {
          const field = this.referenceFieldMap[referenceFieldKey];
          if (field) {
            responses.push({
              referenceFieldKey,
              referenceFieldId: field.referenceFieldId,
              value: map[referenceFieldKey],
              numericValue: null,
              dateValue: '',
              currencyValue: '',
              file: null,
              files: [],
              applicationFormId,
              applicationId: appId
            });
          }
        });
      }

      mapped = responses.reduce((acc, response) => {
        response.value = this.prepareValueForMapping(response);

        return {
          ...acc,
          [response.referenceFieldKey]: response
        };
      }, {});
      this.filterOutResponsesNoLongerOnForm(
        formDefinition,
        mapped
      );
    }
    // get dataset responses here
    tableResponses.forEach((response) => {
      const field = this.referenceFieldMapById[response.tableReferenceFieldId];
      const adaptedRows = response.rows.map((row) => {
        return {
          ...row,
          columns: row.columns.map((col) => {
            return {
              ...col,
              value: this.prepareValueForMapping(col)
            };
          })
        };
      });
      mapped[field.key] = {
        referenceFieldKey: field.key,
        referenceFieldId: field.referenceFieldId,
        value: adaptedRows,
        numericValue: null,
        dateValue: null,
        currencyValue: null,
        file: null,
        files: [],
        applicationFormId,
        applicationId: appId
      };
    });

    this.formatReferenceFieldResponses(
      mapped,
      appId,
      applicationFormId
    );
    const mappedTableResponses: ReferenceFieldsUI.RefResponseMapForAdapting = {};
    Object.keys(mapped).forEach((key) => {
      const field = this.referenceFieldMap[key];
      if (
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset
      ) {
        mappedTableResponses[key] = mapped[key];
      }
    });

    const mapToReturn: ReferenceFieldsUI.RefResponseMap = {};
    Object.keys(mapped).forEach((key) => {
      const field = this.referenceFieldMap[key];
      mapToReturn[field.key] = mapped[key].value;
    });

    if (!skipAddTablesToState) {
      this.setApplicationFormTableRowsMap(
        applicationFormId || revisionId,
        mapToReturn
      );
    }

    return mapToReturn;
  }

  prepareValueForMapping (
    response: ReferenceFieldAPI.ApplicationRefFieldResponse
  ): FormioAnswerValues {
    let value: FormioAnswerValues = response.value || '';
    const field = this.referenceFieldMap[response.referenceFieldKey];
    const isNumber = [
      ReferenceFieldsUI.ReferenceFieldTypes.Aggregate,
      ReferenceFieldsUI.ReferenceFieldTypes.Number,
      ReferenceFieldsUI.ReferenceFieldTypes.DataPoint
    ].includes(field?.type);
    if (isNumber) {
      value = response.numericValue ?? response.value;
    } else if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Date) {
      value = response.dateValue || (response.value as string) || '';
      if (value) {
        value = this.timezoneService.returnMidnightUTCDate(value);
      }
    }

    return value;
  }

  async getTableResponses (
    appId: number,
    appFormId: number,
    tableIds?: number[]
  ) {
    const tableResponses: ReferenceFieldsUI.TableResponseForUi[] = [];
    if (appId && appFormId) {
      if (tableIds?.length > 0) {
        await Promise.all(tableIds.map(async (tableId) => {

          const responses = await this.referenceFieldsResources.getTableResponses(
            appId,
            appFormId,
            tableId
          );

          tableResponses.push({
            tableReferenceFieldId: tableId,
            rows: responses.map<ReferenceFieldsUI.TableResponseRowForUi>((response) => {
              return {
                rowId: response.rowId,
                columns: response.columns
              };
            })
          });
        }));
      }
    }

    return tableResponses;
  }

  formatReferenceFieldResponses (
    responseMap: ReferenceFieldsUI.RefResponseMapForAdapting = {},
    applicationId: number,
    applicationFormId: number
  ) {
    Object.keys(responseMap).forEach((key) => {
      const field = this.referenceFieldMap[key];
      const response = responseMap[key];
      responseMap[key].value = this.formatFieldForUi(
        field,
        response,
        applicationId,
        applicationFormId
      );
      const isTableOrSubset = [
        ReferenceFieldsUI.ReferenceFieldTypes.Table,
        ReferenceFieldsUI.ReferenceFieldTypes.Subset
      ].includes(field.type);
      if (isTableOrSubset) {
        const rows = response.value as ReferenceFieldsUI.TableResponseRowForUi[];
        rows?.forEach((row) => {
          row.columns.forEach((column) => {
            const columnRefField = this.referenceFieldMapById[column.referenceFieldId];
            column.value = this.prepareValueForMapping(column);
            column.value = this.formatFieldForUi(
              columnRefField,
              column,
              applicationId,
              applicationFormId
            );
          });
        });
      }
    });
  }

  convertPipeSeparatedAnswersToArray (response: string) {
    return response.split(/(?!\\)\|/).map((item, i, arr) => {
      if (item && item.endsWith('\\')) {
        arr[i + 1] = item.slice(0, -1) + '|' + arr[i + 1];
        item = null;
      }

      return item;
    }).filter(item => item !== null);
  }

  convertPipeSeparatedAnswersToString (value: string[]) {
    return (value || []).map((item) => {
      return item.replace(/\|/g, '\\|');
    }).join('|');
  }

  formatFieldForUi (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    response: ReferenceFieldAPI.ApplicationRefFieldResponse,
    applicationId: number,
    applicationFormId: number
  ) {
    const value = response.value;
    let adaptedValue = value;
    // All reference field responses are stored as a string,
    // so parse to get boolean value for checkbox
    // and parse to get array for any that support multiple
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Checkbox) {
      adaptedValue = this.convertRefValueToBool(value);
    } else if (field.supportsMultiple) {
      if (value) {
        switch (field.type) {
          case ReferenceFieldsUI.ReferenceFieldTypes.TextArea:
          case ReferenceFieldsUI.ReferenceFieldTypes.TextField:
          case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
            // For text fields with multiple, we separate the answers with a pipe
            adaptedValue = this.convertPipeSeparatedAnswersToArray(value as string);
            if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload) {
              adaptedValue = this.getFilesFromApplicationRefFieldResponse(
                response.files,
                applicationId,
                applicationFormId
              );
            } else {
              if (adaptedValue.length === 0) {
                // Text fields and areas with multiple require at least one item so input shows
                adaptedValue = [''];
              }
            }
            break;
          default:
            // Everything else multiple, we separate with a comma
            adaptedValue = (value as string).split(',').filter(item => item);
            break;
        }
      } else {
        adaptedValue = [];
      }
    } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload) {
      adaptedValue = this.getFilesFromApplicationRefFieldResponse(
        [response.file],
        applicationId,
        applicationFormId
      );
    } else if (
      field.type === ReferenceFieldsUI.ReferenceFieldTypes.DataPoint ||
      field.type === ReferenceFieldsUI.ReferenceFieldTypes.Number
    ) {
      adaptedValue = this.convertRefValueToNumber(value);
    } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Currency) {
      const amount = response.numericValue ?? response.value as number;
      const currency = response.currencyValue;
      adaptedValue = {
        amountInDefaultCurrency: amount,
        amountEquivalent: amount,
        amountForControl: amount,
        currency
      };
    }

    return adaptedValue;
  }

  getFilesFromApplicationRefFieldResponse (
    files: ReferenceFieldAPI.ApplicationRefFieldFile[],
    applicationId: number,
    applicationFormId: number
  ): YcFile[] {
    return (files || []).filter((file) => {
      return !!file?.value;
    }).map((file) => {
      const fileUrl = this.applicationFileService.convertParamsToApplicationFileUrl(
        applicationId,
        applicationFormId,
        file.fileId,
        file.fileName
      );

      return new YcFile(
        file.fileName,
        null,
        fileUrl,
        file.fileId
      );
    });
  }

  convertRefValueToNumber (response: FormioAnswerValues) {
    return (response || response === 0) ? parseFloat(response as string) : null;
  }

  convertRefValueToBool (response: FormioAnswerValues) {
    if (response) {
      return JSON.parse(response as string);
    } else {
      return false;
    }
  }

  mapReferenceFieldResponsesForAPI (
    responses: ReferenceFieldsUI.RefResponseMap,
    isManagerForm: boolean
  ): ReferenceFieldAPI.ApplicationRefFieldResponseForApi[] {
    return Object.keys(responses || {}).filter((referenceFieldKey) => {
      const field = this.referenceFieldMap[referenceFieldKey];
      if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
        return false;
      }
      if (isManagerForm) {
        return field.formAudience === FormAudience.MANAGER;
      }

      return true;
    }).map((referenceFieldKey) => {
      const {
        value,
        dateValue,
        currencyValue
      } = this.adaptResponseValueForApi(
        referenceFieldKey,
        responses[referenceFieldKey]
      );

      const returnVal: ReferenceFieldAPI.ApplicationRefFieldResponseForApi = {
        referenceFieldKey,
        referenceFieldId: this.referenceFieldMap[referenceFieldKey]?.referenceFieldId,
        value: value ?? '',
        numericValue: null,
        dateValue,
        currencyValue
      };

      return returnVal;
    });
  }

  adaptResponseDateValueForApi (
    key: string,
    value: FormioAnswerValues
  ): string {
    // dateValue should be null for all non-dates
    const field = this.referenceFieldMap[key];
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Date) {
      // value will be string if date type
      if (this.isMoment(value)) {
        const midnightUTCDate = this.timezoneService.returnMidnightUTCDate(value.format());

        return midnightUTCDate;
      }
    }

    return null;
  }

  adaptResponseValueForApi (
    key: string,
    value: FormioAnswerValues
  ): {
    value: string;
    dateValue: string;
    currencyValue: string;
  } {
    const dateValue = this.adaptResponseDateValueForApi(key, value);
    const isCurrency = this.referenceFieldMap[key].type === ReferenceFieldsUI.ReferenceFieldTypes.Currency;
    let currencyValue = '';
    if (isCurrency) {
      currencyValue = (value as CurrencyValue)?.currency;
    }
    // Before we send to API, convert all responses to strings
    const field = this.referenceFieldMap[key];
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload) {
      const fileUrls = (value as YcFile<File>[] || []).map((file) => {
        return file.fileUrl;
      });

      value = this.convertPipeSeparatedAnswersToString(fileUrls);
    } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Checkbox) {
      value = value ? 'true' : 'false';
    } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Currency) {
      value = (value as CurrencyValue)?.amountForControl;
    } else if (value instanceof Array) {
      if (
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextArea ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextField
      ) {
        value = this.convertPipeSeparatedAnswersToString(value as string[]);
      } else {
        const sorted = this.arrayHelper.sort(
          ((value as string[]) || [])
        );

        value = sorted.join(',');
      }
    } else if (
      field.type === ReferenceFieldsUI.ReferenceFieldTypes.Date &&
      this.isMoment(value)
    ) {
      value = value.toISOString();
    }

    return {
      value: ((value ?? '') as string).toString(),
      dateValue,
      currencyValue
    };
  }

  mapTableResponsesForAPI (
    responses: ReferenceFieldsUI.RefResponseMap,
    applicationFormId: number,
    applicationId: number,
    revisionId: number,
    isNew: boolean /* If new (offline only), we track off revisionId */
    /* because applicationFormId is not defined yet */
  ): ReferenceFieldAPI.TableChangeResponse {
    let updates: ReferenceFieldAPI.TableChangeValues[] = [];
    let deletions: ReferenceFieldAPI.TableDeletion[] = [];
    Object.keys(responses).forEach((key) => {
      const tableId = this.referenceFieldMap[key].referenceFieldId;
      const oldTable = this.applicationFormTableRowsMap[
        this.getTableFormKey(
          isNew ? revisionId : applicationFormId,
          tableId
        )
      ];
      if (oldTable) {
        const newTable = responses[key] as ReferenceFieldsUI.TableResponseRowForUi[] ?? [];
        const changes = this.doChangeTrackingForTableRows(
          oldTable,
          newTable,
          applicationFormId,
          applicationId,
          tableId
        );

        if (changes.updates.length > 0) {
          updates = [
            ...updates,
            ...changes.updates
          ];
        }
        if (changes.deletions.length > 0) {
          deletions = [
            ...deletions,
            ...changes.deletions
          ];
        }
      }
    });

    return {
      updates,
      deletions
    };
  }

  doChangeTrackingForTableRows (
    oldTable: ReferenceFieldsUI.TableResponseRowForUi[],
    newTable: ReferenceFieldsUI.TableResponseRowForUi[],
    applicationFormId: number,
    applicationId: number,
    tableReferenceFieldId: number
  ): ReferenceFieldAPI.TableChangeResponse {
    const updates: ReferenceFieldAPI.TableChangeValues[] = [];
    const deletions: ReferenceFieldAPI.TableDeletion[] = [];
    /* Check for updates and additions */
    newTable.forEach((newRow, index) => {
      /* Add new row scenario */
      if (!newRow.rowId) {
        updates.push(this.formatTableRowUpdate(
          newRow,
          null,
          tableReferenceFieldId,
          true,
          index
        ));
      } else {
        const valuesInNew = this.mapRowValues(newRow);
        /* Check for updates to values in row */
        const foundOld = oldTable.find((oldRow) => {
          return oldRow.rowId === newRow.rowId;
        });
        let hasUpdates = false;
        if (foundOld) {
          const valuesInOld = this.mapRowValues(foundOld);
          Object.keys(valuesInNew).forEach((id) => {
            const newValue = valuesInNew[+id];
            const oldValue = valuesInOld[+id];
            if (newValue !== oldValue) {
              hasUpdates = true;
            }
          });
          if (hasUpdates) {
            updates.push(this.formatTableRowUpdate(
              newRow,
              newRow.rowId,
              tableReferenceFieldId,
              false,
              index
            ));
          }
        }
      }
    });

    /* Check for deletions */
    oldTable.forEach((oldRow) => {
      const foundNew = newTable.find((newRow) => {
        return newRow.rowId === oldRow.rowId;
      });
      /* If new row not found, it was deleted */
      if (!foundNew) {
        deletions.push({
          applicationId,
          applicationFormId,
          tableReferenceFieldId,
          rowId: oldRow.rowId
        });
      }
    });

    return {
      updates,
      deletions
    };
  }

  formatTableRowUpdate (
    newRow: ReferenceFieldsUI.TableResponseRowForUi,
    rowId: number,
    tableReferenceFieldId: number,
    isNewRecord: boolean,
    index: number
  ): ReferenceFieldAPI.TableChangeValues {
    return {
      rowId,
      isNewRecord,
      index,
      tableReferenceFieldId,
      values: newRow.columns
        .map<ReferenceFieldAPI.ApplicationRefFieldResponseForApi>((column) => {
          const key = column.referenceFieldKey;
          const {
            value,
            currencyValue,
            dateValue
          } = this.adaptResponseValueForApi(
            key,
            column.value
          );

          return {
            referenceFieldKey: key,
            referenceFieldId: column.referenceFieldId,
            value: value ?? '',
            currencyValue,
            numericValue: null,
            dateValue
          };
        })
    };
  }

  mapRowValues (
    row: ReferenceFieldsUI.TableResponseRowForUi
  ): Record<number, FormioAnswerValues> {
    return row.columns.reduce((acc, col) => {
        return {
          ...acc,
          [col.referenceFieldId]: col.value
        };
      }, {});
  }

  isMoment (value: any): value is moment.Moment {
    return value instanceof moment;
  }

  async copyField (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    skipToaster = false
  ) {
    const copyText = this.i18n.translate(
      'common:textCopy',
      {},
      'Copy'
    );
    const fieldToSave: ReferenceFieldAPI.ReferenceFieldDisplayModel = {
      ...field,
      name: this.guessBasedOnExisting(`${field.name} ${copyText}`, 'name'),
      key: this.guessBasedOnExisting(`copy_${field.key}`, 'key')
    };
    const tableFields = await this.getTableFields(field.referenceFieldId);
    const newField = await this.handleCreateOrUpdateField(
      null,
      fieldToSave,
      tableFields,
      true,
      true
    );
    if (!!newField && !skipToaster) {
      this.notifier.success(this.i18n.translate(
        'FORMS:notificationSuccessCopyFormField',
        {},
        'Successfully copied your form field'
      ));
    }

    return newField;
  }

  async handleCopyComponents (
    components: FormDefinitionComponent[]
  ): Promise<ReferenceFieldsUI.FormFieldComponentCopy[]> {
    const copyInfo: ReferenceFieldsUI.FormFieldComponentCopy[] = [];
    try {
      const copyText = this.i18n.translate(
        'common:textCopy',
        {},
        'Copy'
      );
      const additionalGeneratedKeys: string[] = [];
      const newFields = components
        .map<ReferenceFieldAPI.BulkCreateReferenceField>((component) => {
          const oldKey = this.componentHelper.getRefFieldKeyFromCompType(
            component.type
          );
          const field = this.getReferenceFieldByKey(
            oldKey
          );
          const newKey = this.guessBasedOnExisting(
            `copy_${field.key}`,
            'key',
            0,
            additionalGeneratedKeys
          );
          additionalGeneratedKeys.push(newKey);
          copyInfo.push({
            oldKey,
            newKey
          });

          let dataTableId: number = null;
          if (field.customDataTableGuid) {
            dataTableId = this.customDataTablesService.getCDTIdFromGuid(
              field.customDataTableGuid
            );
          }
          let parentReferenceFieldKey: string = null;
          if (field.parentReferenceFieldId) {
            parentReferenceFieldKey = this.findReferenceFieldById(
              field.parentReferenceFieldId
            )?.key ?? null;
          }
          const formAudience = field.formAudience;

          return {
            id: undefined,
            name: this.guessBasedOnExisting(`${field.name} ${copyText}`, 'name'),
            description: field.description,
            type: field.type,
            key: newKey,
            picklistId: dataTableId,
            parentReferenceFieldKey,
            supportsMultiple: field.supportsMultiple,
            category: !field.categoryId ?
              null :
              {
                id: field.categoryId,
                name: this.categoryNameMap[field.categoryId]
              },
            formAudience,
            aggregateType: field.aggregateType,
            isEncrypted: field.isEncrypted,
            isMasked: field.isMasked,
            formatType: field.formatType,
            isTableField: field.isTableField,
            singleResponse: formAudience === FormAudience.APPLICANT ?
              true :
              field.isSingleResponse ?? false
          };
        });

      await this.referenceFieldsResources.bulkCreateReferenceFields(newFields);
      await this.resetFieldsAndCategories();

      return copyInfo;
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'FORMS:textErrorCopyingFields',
        {},
        'There was an error copying the fields'
      ));

      return null;
    }
  }

  adaptTableInfoForSave (
    allowImport: boolean,
    tableFields: ReferenceFieldsUI.TableFieldForUi[]
  ): ReferenceFieldAPI.TableInfoForCreate {
    if (tableFields.length > 0) {
      return {
        allowImport,
        tableFields: tableFields.map<ReferenceFieldAPI.TableFieldForCreate>((field, index) => {
          return {
            referenceFieldId: field.referenceFieldId,
            isRequired: field.isRequired,
            columnOrder: index,
            showInTable: field.showInTable,
            label: field.label
          };
        })
      };
    }

    return null;
  }

  getAggregateFieldChanges (
    tableFieldName: string,
    tableReferenceFieldId: number,
    tableFields: ReferenceFieldsUI.TableFieldForUi[] = []
  ): {
    fieldsToCreateOrUpdate: ReferenceFieldAPI.CreateUpdateReferenceField[];
    fieldsToRemove: number[];
  } {
    const fieldsToCreateOrUpdate: ReferenceFieldAPI.CreateUpdateReferenceField[] = [];
    const fieldsToRemove: number[] = [];
    const pendingKeysToBeCreated: string[] = [];
    tableFields.forEach((field) => {
      const existingAggregate = this.findAggregateForTableColumn(
        tableReferenceFieldId,
        field.referenceFieldId
      );
      if (field.summarizeData) {
        /* Check if table aggreate field has already been created */
        /* Or label has been updated */
        const needToCreate = !existingAggregate;
        const needToUpdate = existingAggregate &&
          existingAggregate.name !== field.summarizeLabel;
        const refIdToUpdate = needToUpdate ? existingAggregate.referenceFieldId : null;
        if (needToCreate || needToUpdate) {
          const key = this.guessKey(
            field.summarizeLabel,
            pendingKeysToBeCreated
          );
          pendingKeysToBeCreated.push(key);
          const relatedField = this.referenceFieldMapById[field.referenceFieldId];
          fieldsToCreateOrUpdate.push({
            referenceFieldId: refIdToUpdate,
            customDataTableGuid: null,
            tableInfo: null,
            name: field.summarizeLabel,
            description: `For: Table: ${tableFieldName}, Field: ${relatedField.name}`,
            type: ReferenceFieldsUI.ReferenceFieldTypes.Number,
            key,
            formatType: field.referenceField.formatType,
            supportsMultiple: false,
            categoryId: null,
            formAudience: FormAudience.APPLICANT,
            parentReferenceFieldId: field.referenceFieldId,
            aggregateType: ReferenceFieldAPI.ReferenceFieldAggregateType.Sum,
            isSingleResponse: false,
            isEncrypted: false,
            isMasked: false,
            isTableField: true,
            aggregateTableReferenceFieldId: tableReferenceFieldId,
            subsetCollectionType: null
          });
        }
      } else if (existingAggregate) {
        /* Aggregate exists, but we are no longer aggregating so we need to delete */
        fieldsToRemove.push(existingAggregate.referenceFieldId);
      }
    });

    return {
      fieldsToCreateOrUpdate,
      fieldsToRemove
    };
  }

  async handleCreateOrUpdateField (
    referenceFieldId: number,
    field: ReferenceFieldAPI.ReferenceFieldBaseModel,
    tableFields: ReferenceFieldsUI.TableFieldForUi[] = [],
    skipToaster = false,
    isCopy = false
  ): Promise<ReferenceFieldAPI.ReferenceFieldDisplayModel> {
    const tableInfo = this.adaptTableInfoForSave(field.tableAllowsImport, tableFields);
    const payload: ReferenceFieldAPI.CreateUpdateReferenceField = {
      referenceFieldId,
      customDataTableGuid: field.customDataTableGuid,
      tableInfo,
      name: field.name,
      description: field.description,
      type: field.type,
      key: field.key,
      supportsMultiple: field.supportsMultiple,
      categoryId: field.categoryId || null,
      formAudience: field.formAudience,
      parentReferenceFieldId: field.parentReferenceFieldId,
      aggregateType: field.aggregateType,
      isSingleResponse: field.isSingleResponse,
      isEncrypted: field.isEncrypted,
      isMasked: field.isMasked,
      isTableField: field.isTableField,
      formatType: field.formatType,
      aggregateTableReferenceFieldId: field.aggregateTableReferenceFieldId,
      subsetCollectionType: field.subsetCollectionType
    };
    const createdField = await this.createOrUpdateField(
      referenceFieldId,
      payload,
      skipToaster,
      isCopy
    );
    if (createdField) {
      const aggregateInfo = this.getAggregateFieldChanges(
        field.name,
        createdField.referenceFieldId,
        tableFields
      );
      await Promise.all(aggregateInfo.fieldsToCreateOrUpdate.map((aggregateField) => {
        return this.createOrUpdateField(
          aggregateField.referenceFieldId,
          aggregateField,
          true,
          isCopy
        );
      }));
      await Promise.all(aggregateInfo.fieldsToRemove.map((idToRemove) => {
        return this.referenceFieldsResources.removeReferenceField(
          idToRemove
        );
      }));
      if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
        this.resetTableColumnsMap(createdField.referenceFieldId);
      } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
         this.resetDataPointsMap(createdField.referenceFieldId);
      }
    }

    return createdField;
  }

  async createOrUpdateField (
    referenceFieldId: number,
    payload: ReferenceFieldAPI.CreateUpdateReferenceField,
    skipToaster = false,
    isCopy = false
  ) {
    try {
      const response = await this.referenceFieldsResources.createOrUpdateField(
        referenceFieldId,
        payload
      );
      await this.resetFieldsAndCategories();
      if (!skipToaster) {
        this.notifier.success(this.i18n.translate(
          'FORMS:notificationSuccessSavedFormField',
          {},
          'Successfully saved your form field'
        ));
      }

      return response;
    } catch (e) {
      this.notifier.error(this.i18n.translate(
        isCopy ?
          'FORMS:notificationErrorCopyFormField' :
          'FORMS:notificationErrorSavingFormField',
        {},
        isCopy ?
          'There was an error copying your form field' :
          'There was an error saving your form field'
      ));
      this.logger.error(e);

      return null;
    }
  }

  setAllReferenceFields (
    fields: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ) {
    this.set('allReferenceFields', fields);
    this.setReferenceFieldMaps();
    this.setAllTypes();
  }

  async getReferenceFieldsPaginated (
    paginationOptions: PaginationOptions<ReferenceFieldAPI.ReferenceFieldDisplayModel>
  ) {
    const formattedResponse = this.formatPaginationOptions(paginationOptions);
    const response = await this.referenceFieldsResources.searchReferenceFields(
      formattedResponse.paginationOptions,
      formattedResponse.formIds
    );
    response.records = this.adaptReferenceFields(response.records);

    return {
      success: true,
      data: {
        recordCount: response.recordCount,
        records: response.records
      }
    };
  }

  formatPaginationOptions (
    paginationOptions: PaginationOptions<ReferenceFieldAPI.ReferenceFieldDisplayModel>
  ): {
    paginationOptions: PaginationOptions<ReferenceFieldAPI.ReferenceFieldDisplayModel>;
    formIds: number[];
  } {
    const formIdsFilterIndex = paginationOptions.filterColumns.findIndex((col) => {
      return col.columnName === 'formIds';
    });
    const formIdsFilter = paginationOptions.filterColumns[formIdsFilterIndex];
    if (formIdsFilterIndex > -1) {
      paginationOptions = {
        ...paginationOptions,
        filterColumns: [
          ...paginationOptions.filterColumns.slice(0, formIdsFilterIndex),
          ...paginationOptions.filterColumns.slice(formIdsFilterIndex + 1)
        ]
      };
    }
    let formIds: number[];
    if (formIdsFilter) {
      formIds = formIdsFilter.filters.map((filter) => {
        return +filter.filterValue;
      });
    }

    return {
      paginationOptions,
      formIds
    };
  }

  setAllTypes () {
    const applicantTypes: ReferenceFieldsUI.TypeForFormio[] = [];
    const managerTypes: ReferenceFieldsUI.TypeForFormio[] = [];
    const allTypeOptions = this.getReferenceFieldTypeOptions();
    allTypeOptions.forEach((type) => {
      const audiences = this.getAudiencesForRefFieldType(type.value);
      if (audiences.APPLICANT) {
        applicantTypes.push({
          type: type.value,
          name: type.label,
          formAudience: FormAudience.APPLICANT
        });
      }
      if (audiences.MANAGER) {
        managerTypes.push({
          type: type.value,
          name: type.label,
          formAudience: FormAudience.MANAGER
        });
      }
    });
    const allTypes = [
      ...applicantTypes,
      ...managerTypes
    ];
    this.set('allTypes', allTypes);
  }

  async fetchAllFields () {
    if (!this.allReferenceFields) {
      const { records } = await this.referenceFieldsResources.searchReferenceFields({
        returnAll: true,
        retrieveTotalRecordCount: false,
        filterColumns: [],
        orFilterColumns: [],
        sortColumns: [],
        rowsPerPage: 10,
        pageNumber: 0
      });

      const adaptedRecords = this.adaptReferenceFields(records);
      this.setAllReferenceFields(adaptedRecords);
    }
  }

  adaptReferenceFields (records: ReferenceFieldAPI.ReferenceFieldDisplayModel[]) {
    return records.map<ReferenceFieldAPI.ReferenceFieldDisplayModel>((record) => {
      let type = record.type;

      if (
        record.type === ReferenceFieldsUI.ReferenceFieldTypes.Number &&
        record.aggregateType
      ) {
        // the API stores aggregate reference fields as numbers
        // we need to make the UI aware of this
        type = ReferenceFieldsUI.ReferenceFieldTypes.Aggregate;
      }

      let categoryId = record.categoryId;

      if (!categoryId) {
        // This will represent No category or "Other"
        categoryId = 0;
      }

      let formAudience = record.formAudience;

      if (!record.formAudience) {
        formAudience = FormAudience.APPLICANT;
      }

      let supportsMultiple = record.supportsMultiple;


      if (record.type === ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes) {
        supportsMultiple = true;
      }

      return {
        ...record,
        type,
        categoryId,
        formAudience,
        supportsMultiple
      };
    });
  }

  async fetchAllCategories () {
    if (!this.categories) {
      const categories = await this.referenceFieldsResources.getCategories();
      const map: SimpleStringMap<string> = {
        0: this.i18n.translate(
          'common:textOther',
          {},
          'Other'
        )
      };
      const categoryOptions: TypeaheadSelectOption[] = [];
      categories.forEach((category) => {
        map[category.id] = category.name;
        categoryOptions.push({
          label: category.name,
          value: category.id
        });
      });
      this.set('categories', this.arrayHelper.sort(categories, 'name'));
      this.set('categoryNameMap', map);
      this.set('categoryOptions', this.arrayHelper.sort(categoryOptions, 'label'));
    }
  }

  setReferenceFieldMaps () {
    const keyMap: SimpleStringMap<ReferenceFieldAPI.ReferenceFieldDisplayModel> = {};
    const idMap: SimpleNumberMap<ReferenceFieldAPI.ReferenceFieldDisplayModel> = {};
    const isParentMap: Record<number, boolean> = {};
    this.allReferenceFields.forEach((field) => {
      keyMap[field.key] = field;
      idMap[field.referenceFieldId] = field;
      const isParent = this.getIsParentRefField(field.referenceFieldId);
      isParentMap[field.referenceFieldId] = isParent;
    });

    this.set('referenceFieldMap', keyMap);
    this.set('referenceFieldMapById', idMap);
    this.set('isParentRefFieldMap', isParentMap);
  }


  async fetchFieldsByFormRevisionId (
    formRevisionId: number
  ): Promise<ReferenceFieldAPI.ReferenceFieldDisplayModel[]> {
    if (formRevisionId) {
      return this.referenceFieldsResources.getReferenceFieldsByFormRevisionId(
        formRevisionId
      );
    }

    return [];
  }

  async removeField (fieldId: number) {
    try {
      await this.referenceFieldsResources.removeReferenceField(fieldId);
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'FORMS:notificationSuccessDeletingFormField',
        {},
        'Successfully deleted your form field'
      ));
    } catch (e) {
      this.notifier.error(this.i18n.translate(
        'FORMS:notificationErrorDeletingFormField',
        {},
        'There was an error deleting your form field'
      ));
      this.logger.error(e);
    }
  }

  doesTypeHaveOptions (type: ReferenceFieldsUI.ReferenceFieldTypes) {
    return [
      ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable,
      ReferenceFieldsUI.ReferenceFieldTypes.Radio,
      ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes
    ].includes(type);
  }

  getSupportsMultipleSeparator (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel
  ) {
    if (!field || !field.supportsMultiple) {
      return '';
    }

    switch (field.type) {
      case ReferenceFieldsUI.ReferenceFieldTypes.TextArea:
      case ReferenceFieldsUI.ReferenceFieldTypes.TextField:
      case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
        return '|';
      default:
        return ',';
    }
  }

  extractReferenceFieldsFromForm (
    definitions: FormDefinitionForUi[]
  ): {
      referenceFieldIds: number[];
      customDataTableGuids: string[];
      referenceFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[];
    } {
    const referenceFieldIds: number[] = [];
    const customDataTableGuids: string[] = [];
    const referenceFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[] = [];
    definitions.forEach((definition) => {
      this.componentHelper.eachComponent(definition.components, (component) => {
        if (this.componentHelper.isReferenceFieldComp(component.type)) {
          const key = this.componentHelper.getRefFieldKeyFromCompType(component.type);
          const found = this.get('allReferenceFields').find((field) => {
            return field.key === key;
          });
          if (found) {
            referenceFieldIds.push(found.referenceFieldId);
            referenceFields.push(found);
            const dataTableGuid = found.customDataTableGuid;
            if (dataTableGuid) {
              customDataTableGuids.push(dataTableGuid);
            }
          }
        }
      });
    });

    return {
      referenceFieldIds,
      customDataTableGuids: uniq(customDataTableGuids),
      referenceFields
    };
  }

  guessKey (
    name: string,
    additionalGeneratedKeys: string[] = []
  ) {
    const pascal = this.inflect.pascalize(
      (name.match(/[a-z]|[A-Z]|\ |\d/g) || []).join('')
    );
    let guessedKey = pascal ? pascal[0].toLowerCase() + pascal.slice(1) : '';
    const isRootZone = this.clientSettingsService.clientSettings.isRootClient;
    const prependedText = isRootZone ? STANDARD_FIELD_PREFIX : '';
    if (prependedText) {
      guessedKey = prependedText + guessedKey;
    }

    return this.guessBasedOnExisting(guessedKey, 'key', 0, additionalGeneratedKeys);
  }

  private guessBasedOnExisting (
    val: string,
    prop: 'key'|'name',
    incCount = 0,
    additionalGeneratedKeys: string[] = []
  ): string {
    const desired = `${val}${incCount ? `${prop === 'name' ? ' ' : ''}${incCount + 1}` : ''}`;
    const allExisting = this.allReferenceFields.map((field) => field[prop]);
    const allExistingAndAdditional = allExisting.concat(additionalGeneratedKeys);
    const existingKey = allExistingAndAdditional.find(keyOrName => {
      const existing = keyOrName || '';

      return existing.toLowerCase() === desired.toLowerCase();
    });

    if (existingKey) {
      return this.guessBasedOnExisting(
        val,
        prop,
        incCount + 1,
        additionalGeneratedKeys
      );
    }

    return desired;
  }

  async exportReferenceFields (fields: number[]) {
    try {
      const exportedForms = await this.referenceFieldsResources.exportReferenceFields(
        fields
      );
      this.fileService.downloadRaw(
        Base64.encode(JSON.stringify(exportedForms)),
        `ref_fields_export_${moment().format('YYYYMMDDHHmmss')}.bin`
      );
      this.notifier.success(this.i18n.translate(
        'FORMS:textSuccessfullyExportedSelectedFormFields',
        {},
        'Successfully exported the selected form fields'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'FORMS:textErrorExportingSelectedFormFields',
        {},
        'There was an error exporting the selected form fields'
      ));
    }
  }

  async importReferenceFields (fieldExport: string) {
    let fields: ReferenceFieldAPI.ExportReferenceField[];
    try {
      fields = JSON.parse(Base64.decode(fieldExport));
    } catch (e) {
      this.logger.error(e);
      this.notifier.warning(this.i18n.translate(
        'FORMS:notificationInvalidFileType',
        {},
        'The provided file was invalid, please obtain a valid file and try again.'
      ));

      return;
    }
    await this.handleBulkCreateReferenceFields(fields, true);
  }

  async handleBulkCreateReferenceFields (
    fieldsToImport: ReferenceFieldAPI.ExportReferenceField[]|ReferenceFieldAPI.BulkCreateReferenceField[],
    isImport = false
  ) {
    try {
      if (isImport) {
        await this.referenceFieldsResources.importReferenceFields(
          fieldsToImport as ReferenceFieldAPI.ExportReferenceField[]
        );
      } else {
        await this.referenceFieldsResources.bulkCreateReferenceFields(
          fieldsToImport as ReferenceFieldAPI.BulkCreateReferenceField[]
        );
      }
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'FORMS:notificationSuccessFormFields',
        {},
        'Successfully imported your form fields'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'FORMS:notificationErrorImportingFormFields',
        {},
        'There was an error importing your form fields'
      ));
    }
  }

  getParentPicklistComp (
    picklist: CustomDataTable,
    hasParent: boolean
  ) {
    const parentDataTable = this.customDataTablesService.getCDTFromId(
      picklist.parentPicklistId
    );
    const PARENT_PICKLIST: FormDefinitionComponent = hasParent ? {
      key: 'parentPicklist',
      input: true,
      type: 'textfield',
      label: 'Parent picklist',
      dataSrc: 'values',
      defaultValue: parentDataTable.name,
      disabled: true
    } : undefined;

    return PARENT_PICKLIST;
  }

  getHideWithoutParentResponseComp (
    hasParent: boolean
    ) {
    const HIDE_WITHOUT_PARENT_RESPONSE: FormDefinitionComponent = hasParent ? {
      key: 'hideWithoutParentResponse',
      input: true,
      type: 'checkbox',
      label: 'Hide until parent picklist has a response',
      defaultValue: 'true'
    } : undefined;

    return HIDE_WITHOUT_PARENT_RESPONSE;
  }

  getDependencyValidationFromFormDef (formDef: FormDefinitionForUi): {
    childPicklist: ReferenceFieldAPI.ReferenceFieldDisplayModel;
    parentPicklist: ReferenceFieldAPI.ReferenceFieldDisplayModel;
  } {
    // get fields on form from service
    const { referenceFields } = this.extractReferenceFieldsFromForm([formDef]);

    // for each child field, make sure the parent is on the formDef
    // filter out aggregate fields since they can be added to GM forms without parent
    const orphanedChild = referenceFields.find((childToCheck) => {
      return !childToCheck.aggregateType && childToCheck.parentReferenceFieldId && referenceFields.every((potentialParent) => {
        return childToCheck.parentReferenceFieldId !== potentialParent.referenceFieldId;
      });
    });
    // if there is an orphaned child, return the parent as well
    const parentPicklist = orphanedChild ?
      this.allReferenceFields.find((refField) => {
        return refField.referenceFieldId === orphanedChild.parentReferenceFieldId;
      }) :
      null;

    // return child and parent that make this move invalid
    const invalidObj = {
      childPicklist: orphanedChild,
      parentPicklist
    };

    return invalidObj;
  }

  checkForParentPicklistOnForm (parentPicklistId: number): {
    parentIsOnForm: boolean;
    parentPicklist: ReferenceFieldAPI.ReferenceFieldDisplayModel;
  } {
    const parentIsOnForm = this.currentFormRefFields.some((refField) => refField.referenceFieldId === parentPicklistId);
    const parentPicklist = this.allReferenceFields.find((refField) => refField.referenceFieldId === parentPicklistId);

    return {
      parentIsOnForm,
      parentPicklist
    };
  }

  getInvalidDependencyText (
    formField: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    parentFormField: ReferenceFieldAPI.ReferenceFieldDisplayModel
  ) {
    const invalidText = this.i18n.translate(
      'FORMS:textParentPicklistMustBeOnForm',
      {
        parentName: parentFormField.name,
        parentCategory: this.categoryNameMap[parentFormField.categoryId],
        childName: formField.name
      },
      'Parent picklist __parentName__ found in the __parentCategory__ category must be placed on the form prior to adding __childName__'
    );

    return invalidText;
  }

  getEditFormSupportsSettings (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    isForDisplayOnly: boolean
  ) {
    const {
      isCheckbox,
      isRadio,
      isSelectBoxes,
      isNumber,
      isTextArea,
      isTextField,
      isFileUpload,
      isTable,
      isDate,
      isCurrencyField,
      isSubset,
      isAggregateField,
      isExternalApiField
    } = this.getTypeHelpers(field);
    const supportsDefaultValue = !isCheckbox &&
      !field.supportsMultiple &&
      !isSelectBoxes &&
      !isFileUpload &&
      !isAggregateField &&
      !isExternalApiField &&
      !isTable &&
      !isSubset &&
      !isForDisplayOnly;
    const defaultValType = supportsDefaultValue ?
      (isDate ? DefaultValType.Date : DefaultValType.Text) :
      DefaultValType.None;
    const supportsPlaceholder = !isCheckbox &&
      !isRadio &&
      !isSelectBoxes &&
      !isAggregateField &&
      !isCurrencyField &&
      !isTable &&
      !isSubset &&
      !isExternalApiField;
    const supportsWordCount = isTextArea || isTextField;
    const supportsMinMax = (
      isNumber ||
      isTable ||
      isSubset ||
      (isFileUpload && field?.supportsMultiple)
    ) && !isAggregateField && !isForDisplayOnly;
    const supportsDataTab = !((isTextArea || isTextField) && field.supportsMultiple) &&
      !isFileUpload &&
      !isAggregateField &&
      !isTable &&
      !isSubset &&
      !isForDisplayOnly;
    const supportsValidationTab = !isAggregateField && !isExternalApiField;
    const supportsDisabled = !isAggregateField && !isExternalApiField && !isForDisplayOnly;
    const supportsTooltip = !isCurrencyField && !isExternalApiField;
    const supportsClearWhenHidden = !isAggregateField &&
      !isTable &&
      !isSubset &&
      !isForDisplayOnly;
    const supportsHideLabel = !isCheckbox && !isExternalApiField;
    const supportsSetValueTab = !isFileUpload &&
      !isTable &&
      !isAggregateField &&
      !isSubset &&
      !isExternalApiField &&
      !isForDisplayOnly;
    const supportsFormulaBuilder = (isNumber || isCurrencyField) && !isForDisplayOnly;

    return {
      defaultValType,
      supportsDisabled,
      supportsFormulaBuilder,
      supportsMinMax,
      supportsPlaceholder,
      supportsSetValueTab,
      supportsTooltip,
      supportsValidationTab,
      supportsWordCount,
      supportsClearWhenHidden,
      supportsDataTab,
      supportsHideLabel
    };
  }

  getTypeHelpers (field: ReferenceFieldAPI.ReferenceFieldDisplayModel) {
    const isCheckbox = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Checkbox;
    const isRadio = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Radio;
    const isDate = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Date;
    const isSelectBoxes = field.type === ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes;
    const isNumber = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Number;
    const isTextField = field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextField;
    const isTextArea = field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextArea;
    const isFileUpload = field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload;
    const isTable = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table;
    const isAggregateField = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Aggregate;
    const isCurrencyField = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Currency;
    const isSubset = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset;
    const isDependentPicklist = field.customDataTableGuid &&
      field.parentReferenceFieldId;
    const isExternalApiField = field.type === ReferenceFieldsUI.ReferenceFieldTypes.ExternalAPI;

    return {
      isCheckbox,
      isRadio,
      isAggregateField,
      isDate,
      isSelectBoxes,
      isNumber,
      isTextArea,
      isTextField,
      isFileUpload,
      isTable,
      isCurrencyField,
      isSubset,
      isDependentPicklist,
      isExternalApiField
    };
  }

  async bulkUpdateCategory (
    fieldIds: number[],
    categoryId: number
  ) {
    try {
      await this.referenceFieldsResources.bulkUpdateCategory(
        fieldIds,
        categoryId
      );
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'GLOBAL:textSuccessUpdateTheCategory',
        {},
        'Successfully updated the category'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'GLOBAL:textErrorUpdatingTheCategory',
        {},
        'There was an error updating the category'
      ));
    }
  }

  async handleCreateOrEditCategory (
    categoryId: number,
    categoryName: string
  ): Promise<number> {
    try {
      const id = await this.referenceFieldsResources.createOrEditCategory(
        categoryId,
        categoryName
      );
      await this.resetCategories();
      this.notifier.success(this.i18n.translate(
        categoryId ?
          'GLOBAL:textSuccessUpdateTheCategory' :
          'GLOBAL:textSuccessAddingTheCategory',
        {},
        categoryId ?
          'Successfully updated the category' :
          'Successfully added the category'
      ));

      return id;
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        categoryId ?
          'GLOBAL:textErrorUpdatingTheCategory' :
          'common:textErrorAddingNewCategory',
        {},
        categoryId ?
          'There was an error updating the category' :
          'There was an error adding the new category'
      ));

      return null;
    }
  }

  async handleDeleteCategory (categoryId: number) {
    try {
      await this.referenceFieldsResources.deleteCategory(
        categoryId
      );
      await this.resetCategories();
      this.notifier.success(this.i18n.translate(
        'GLOBAL:textSuccessDeleteCategory',
        {},
        'Successfully deleted the category'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'common:textErrorDeletingCategory',
        {},
        'There was an error deleting the category'
      ));
    }
  }

  getUniqueCategoryKey (
    categoryName: string,
    formAudience: FormAudience,
    categoryId: number
  ) {
    return `${categoryName}-${formAudience}-${categoryId}-Category`;
  }

  convertMergeFieldsToOneField (
    fields: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ): ReferenceFieldAPI.ReferenceFieldDisplayModel {
    const mappedName = uniq(fields.map((field) => field.name));
    const mappedDesc = uniq(fields.map((field) => field.description));
    const mappedCategory = uniq(fields.map((field) => field.categoryId));
    const mappedGuids = uniq(fields.map((field) => field.customDataTableGuid));
    const mappedParentFieldIds = uniq(fields.map((field) => field.parentReferenceFieldId));
    const mappedResponseType = uniq(fields.map((field) => field.isSingleResponse));
    const mappedMulti = fields.every((field) => field.supportsMultiple);
    const mappedEncrypted = fields.every(field => field.isEncrypted);
    const mappedMasked = fields.every(field => field.isMasked);
    const mappedTableField = fields.every(field => field.isTableField);
    const mappedFormatType = fields.map(field => field.formatType);

    return {
      name: mappedName.length === 1 ? mappedName[0] : '',
      description: mappedDesc.length === 1 ? mappedDesc[0] : '',
      key: '',
      type: fields[0].type,
      formAudience: fields[0].formAudience,
      categoryId: mappedCategory.length === 1 ? mappedCategory[0] : null,
      customDataTableGuid: mappedGuids.length === 1 ? mappedGuids[0] : '',
      parentReferenceFieldId: mappedParentFieldIds.length === 1 ?
        mappedParentFieldIds[0] :
        null,
      supportsMultiple: mappedMulti,
      aggregateType: null,
      isSingleResponse: mappedResponseType.length === 1 ?
        mappedResponseType[0] :
        false,
      isEncrypted: mappedEncrypted,
      isMasked: mappedMasked,
      tableAllowsImport: false,
      isTableField: mappedTableField,
      aggregateTableReferenceFieldId: null,
      referenceFieldTableId: null,
      formatType: mappedFormatType?.[0],
      standardComponentIsPublished: true,
      isStandardProductField: false,
      subsetCollectionType: null,
      referenceFieldId: null,
      formCount: 0,
      usedOnReports: false,
      createdBy: '',
      createDate: '',
      updatedBy: '',
      updateDate: ''
    };
  }

  async handleMergeFields (
    payload: ReferenceFieldAPI.MergeFormFieldsApi
  ) {
    try {
      await this.referenceFieldsResources.mergeFields(payload);
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'GLOBAL:textSuccessfullyMergedFields',
        {},
        'Successfully merged the form fields'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'common:textErrorMergingFields',
        {},
        'There was an error merging the form fields'
      ));
    }
  }

  getCategoryTokenGroups (
    fieldPrefix = 'ReferenceField',
    usage: AdHocReportingUI.Usage
  ) {
    const map: SimpleStringMap<ReferenceFieldAPI.ReferenceFieldDisplayModel[]> = {};
    this.allReferenceFields.filter((field) => {
      const isSingleResponse = usage === AdHocReportingUI.Usage.TOKENS ?
        field.isSingleResponse :
        true;

      return isSingleResponse &&
        !field.isMasked &&
        !field.isTableField &&
        field.type !== ReferenceFieldsUI.ReferenceFieldTypes.Table &&
        (
          field.formCount > 0 ||
          !!field.aggregateType
        );
      }).forEach((field) => {
        if (!map[field.categoryId]) {
          map[field.categoryId] = [{...field}];
        } else {
          map[field.categoryId] = [
            ...map[field.categoryId],
            field
          ];
        }
      });
    const tokenGroups = Object.keys(map).map((key) => {
      const sortedFields = this.arrayHelper.sort(map[key], 'name');

      return {
        groupDisplay: this.categoryNameMap[key],
        tokens: sortedFields.map((field) => {
          return {
            display: field.name,
            value: `${fieldPrefix}.${field.key}`
          };
        })
      };
    });
    const sortedGroups = this.arrayHelper.sort(tokenGroups, 'groupDisplay');

    return sortedGroups;
  }

  async formatCreateEditFieldModalReturn (
    field: ReferenceFieldAPI.ReferenceFieldBaseModel,
    tableFields: ReferenceFieldsUI.TableFieldForUi[],
    addingCategory = false,
    isSecondarySave = false
  ): Promise<ReferenceFieldsUI.ModalReturn> {
    field = await this.handleFormatAndAddCategory(
      field,
      addingCategory
    );

    return {
      field: {
        ...field
      },
      tableFields,
      isSecondarySave
    };
  }

  async formatMergeModalReturn (
    field: ReferenceFieldAPI.ReferenceFieldBaseModel,
    referenceFieldId1: number,
    referenceFieldId2: number,
    priorityReferenceFieldId: number,
    addingCategory = false
  ): Promise<ReferenceFieldAPI.MergeFormFieldsApi> {
    field = await this.handleFormatAndAddCategory(
      field,
      addingCategory
    );

    return {
      ...field,
      referenceFieldId1,
      referenceFieldId2,
      priorityReferenceFieldId
    };
  }

  async handleFormatAndAddCategory (
    field: ReferenceFieldAPI.ReferenceFieldBaseModel,
    addingCategory = false
  ) {
    let supportsMultiple = field.supportsMultiple;
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes) {
      // checkboxes should always support multiple
      supportsMultiple = true;
    }
    let categoryId = field.categoryId;
    if (addingCategory && field.categoryId) {
      // When adding category, categoryId form control stores the name.
      // Create the category to get the ID
      categoryId = await this.handleCreateOrEditCategory(
        undefined,
        field.categoryId as unknown as string
      );
    }
    if (field.formatType === ReferenceFieldAPI.ReferenceFieldFormattingType.NONE) {
      field.formatType = null;
    }
    // if we are saving a field as an aggregate
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Aggregate) {
      // convert it to a number
      field.type = ReferenceFieldsUI.ReferenceFieldTypes.Number;
      field.formAudience = FormAudience.MANAGER;
      field.isSingleResponse = false;
    } else {
      // otherwise clear out the aggregate setting
      field.aggregateType = null;
      field.isSingleResponse = field.formAudience === FormAudience.APPLICANT ?
        true :
        field.isSingleResponse ?? false;
    }

    return {
      ...field,
      categoryId: categoryId || null,
      supportsMultiple
    };
  }

  getMergeInvalidAlert (
    fieldsToMerge: ReferenceFieldAPI.ReferenceFieldDetail[]
  ) {
    const field1Forms = (fieldsToMerge[0].forms || []).map((form) => {
      return `${form.formId}.${form.formRevisionId}`;
    });
    const field2Forms = (fieldsToMerge[1].forms || []).map((form) => {
      return `${form.formId}.${form.formRevisionId}`;
    });
    const overlap = intersection(field1Forms, field2Forms);
    if (overlap.length > 0) {
      let formsString = '';
      const formsList = this.arrayHelper.sort(overlap.map((form) => {
        return this.translationService.viewTranslations.FormTranslation[
          form.split('.')[0]
        ]?.Name;
      }));
      formsList.forEach((formName) => {
        formsString += `<li>
          ${formName}
        </li>`;
      });
      const invalidFormMessage = this.i18n.translate(
        'FORMS:textCannotMergeFieldsBecauseOnSameForm',
        {},
        'The selected fields cannot be merged because they are on the same form. Form fields can only exist on one form at a time. Merging these fields would result in an invalid form.'
      );

      return `
        ${invalidFormMessage}
        <br>
        ${formsString}
      `;
    }

    return '';
  }

  getDetailForField (referenceFieldId: number) {
    if (referenceFieldId) {
      return this.referenceFieldsResources.getReferenceFieldDetail(
        referenceFieldId
      );
    }

    return null;
  }

  getDetailForFields (
    fieldsToMerge: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ) {
    const translations = this.translationService.viewTranslations.FormTranslation;

    return Promise.all(fieldsToMerge.map(async (field) => {
      const detail = await this.referenceFieldsResources.getReferenceFieldDetail(
        field.referenceFieldId
      );
      detail.forms = detail.forms || [];
      detail.forms.forEach((form) => {
        form.name = translations[form.formId]?.Name || form.name;
      });
      detail.forms = this.arrayHelper.sort(detail.forms, 'name');

      return detail;
    }));
  }

  async getApplicationResponsesForMerge (
    referenceFieldId1: number,
    referenceFieldId2: number
  ): Promise<ReferenceFieldAPI.ApplicationResponse[]> {
    const responses = await this.referenceFieldsResources.getApplicationResponsesForMerge(
      referenceFieldId1,
      referenceFieldId2
    );
    const field1 = this.findReferenceFieldById(referenceFieldId1);
    if (field1.customDataTableGuid) {
      await this.customDataTablesService.setCustomDataTableOptionsFromGuid(
        field1.customDataTableGuid,
        true,
        this.userService.getCurrentUserCulture()
      );
      const options = this.dataTables[field1.customDataTableGuid];
      responses.forEach((response) => {
        const found1 = options.find((opt) => {
          return opt.key === response.referenceField1Response;
        });
        response.referenceField1Response = found1?.value ??
          response.referenceField1Response;
        const found2 = options.find((opt) => {
          return opt.key === response.referenceField2Response;
        });
        response.referenceField2Response = found2?.value ??
          response.referenceField2Response;
      });
    }

    return responses;
  }

  /**
   *
   * @param recordIds This could be formIds or a tableIds. Used for getting components off the componentMap.
   * @param componentMap This is used to building columns based on related components.
   * @param usage Ad hoc, dashboards, or tokens.
   * @param rootObject Object the report is based on.
   */
  getCategoryMapFromRecordIds (
    recordIds: number[],
    componentMap: SimpleNumberMap<AdHocReportingUI.FormComponentSummary[]>,
    usage = AdHocReportingUI.Usage.AD_HOC,
    rootObject?: AdHocReportingUI.RootObject<string>
  ): SimpleStringMap<ReferenceFieldAPI.ReferenceFieldAdHocModel[]> {
    let skipAggregates = usage === AdHocReportingUI.Usage.TOKENS;
    if (rootObject) {
      skipAggregates = skipAggregates || !!rootObject.noFormLogic;
    }
    const categoryMap: SimpleStringMap<
      ReferenceFieldAPI.ReferenceFieldAdHocModel[]
    > = {};
    if (!skipAggregates) {
      // TODO: remove once we allow manager forms on other reports
      // for now, we will show all aggregate fields on ad hoc reports
      const aggregateFields = this.allReferenceFields.filter(field => {
        return field.type === ReferenceFieldsUI.ReferenceFieldTypes.Aggregate;
      });

      aggregateFields.forEach(field => {
        this.addFieldToCategoryMap(field, categoryMap, null);
      });
    }

    const isCustomFormReport = rootObject?.property === 'formData';
    const isTable = rootObject.type === AdHocReportingAPI.AdHocReportModelType.Table;
    const allowMultipleResponseFields = isCustomFormReport || isTable;
    recordIds.forEach((recordId) => {
      return componentMap[recordId]?.map((comp) => {
        const key = comp.referenceFieldKey ?? this.componentHelper.getRefFieldKeyFromCompType(comp.type);
        let field = this.getReferenceFieldByKey(key);
        if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.DataPoint) {
          field = {
            ...field,
            referenceFieldTableId: comp.referenceFieldTableId
          };
        }
        const shouldAddField = this.shouldAddFieldToCategoryMap(
          field,
          false,
          allowMultipleResponseFields,
          rootObject
        );

        if (shouldAddField) {
          this.handleAddToMap(
            field,
            categoryMap,
            recordId
          );
        }
      });
    });

    // Add standard fields to map
    if (rootObject.type !== AdHocReportingAPI.AdHocReportModelType.Budgets) {
      const tableFieldIds: number[] = [];
      if (isTable) {
        recordIds.forEach((tableId) => {
          this.tableColumnsMap[tableId]?.forEach((_field) => {
            tableFieldIds.push(_field.referenceFieldId);
          });
        });
      }
      const standardFields = this.getStandardFields();
      standardFields.filter((field) => {
        if (isTable) {
          return tableFieldIds.includes(field.referenceFieldId);
        }

        return field.type !== ReferenceFieldsUI.ReferenceFieldTypes.Table &&
          field.type !== ReferenceFieldsUI.ReferenceFieldTypes.Subset;
      }).forEach((field) => {
        const shouldAddField = this.shouldAddFieldToCategoryMap(
          field,
          true,
          allowMultipleResponseFields,
          rootObject
        );
        if (shouldAddField) {
          this.handleAddToMap(
            field,
            categoryMap,
            null
          );
        }
      });
    }

    return categoryMap;
  }

  shouldAddFieldToCategoryMap (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    allowStandardFields: boolean,
    allowMultipleResponseFields: boolean,
    rootObject: AdHocReportingUI.RootObject<string>
  ) {
    const isSingleResponse = field?.formAudience === FormAudience.APPLICANT ||
      field?.isSingleResponse;
    const isTableReport = rootObject.type === AdHocReportingAPI.AdHocReportModelType.Table;
    const isStandardField = field?.isStandardProductField;
    const isTableTypeField = field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Table
      || field?.isTableField;

    return field &&
      field.type !== ReferenceFieldsUI.ReferenceFieldTypes.Subset &&
      (!isStandardField || allowStandardFields) &&
      (isSingleResponse || allowMultipleResponseFields) &&
      (!isTableTypeField || isTableReport);
  }

  handleAddToMap (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    categoryMap: SimpleStringMap<ReferenceFieldAPI.ReferenceFieldAdHocModel[]>,
    recordId: number
  ) {
    this.addFieldToCategoryMap(field, categoryMap, recordId);
    // search for a related field that has this field as a parent
    const childField = (this.allReferenceFields || []).find((_field) => {
      return _field.parentReferenceFieldId === field.referenceFieldId;
    });
    // and if there is a child and it is an aggregate field
    if (childField?.aggregateType) {
      // add the it to the bucket(s)
      this.addFieldToCategoryMap(childField, categoryMap, recordId);
    }
  }

  private addFieldToCategoryMap (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    categoryMap: SimpleStringMap<ReferenceFieldAPI.ReferenceFieldAdHocModel[]>,
    recordId: number
  ) {
    const categoryId = field.isStandardProductField ?
      STANDARD_FIELDS_CATEGORY_ID :
      field.categoryId;
    if (categoryMap[categoryId]) {
      const index = categoryMap[categoryId].findIndex((f) => {
        return f.key === field.key;
      });
      if (index > -1) {
        const categoryItem = categoryMap[categoryId][index];
        // If exists, update with additional formId
        categoryMap[categoryId] = [
          ...categoryMap[categoryId].slice(0, index),
          {
            ...categoryItem,
            formIds: recordId ? [
              ...categoryItem.formIds,
              recordId
            ] : categoryItem.formIds
          },
          ...categoryMap[categoryId].slice(index + 1)
        ];
      } else {
        // else, add new field with formId
        categoryMap[categoryId] = [
          ...categoryMap[categoryId],
          {
            ...field,
            formIds: recordId ? [recordId] : []
          }
        ];
      }
    } else {
      categoryMap[categoryId] = [{
        ...field,
        formIds: recordId ? [recordId] : []
      }];
    }
  }

  getStandardFields () {
    return this.allReferenceFields.filter((field) => {
      return field.isStandardProductField;
    });
  }

  /**
   * Get the form fields available given a form's audience, field's audience, and field's type
   *
   * @param type Type of the reference field
   * @param fieldAudience Audience for the field
   * @param formAudience Audience for the form the field is being added to
   */
  getFormFieldsByTypeAndAudience (
    type: ReferenceFieldsUI.ReferenceFieldTypes,
    fieldAudience: FormAudience,
    formAudience: FormAudience,
    onlyReturnPublished = false // used for Root zone scenario
  ): ReferenceFieldAPI.ReferenceFieldDisplayModel[] {
    const currentFormRefFieldKeys = (this.currentFormRefFields || []).map((refField) => {
      return refField.key;
    });
    const addingManagerFieldToApplicantForm = formAudience === FormAudience.APPLICANT && fieldAudience === FormAudience.MANAGER;

    return this.allReferenceFields.filter((field) => {
      // allow this field to be selected if
      // the field's type matches
      return (field.type === type) &&
        // the field's audience matches
        (field.formAudience === fieldAudience) &&
        // and the field does not belong on a table field
        (!field.isTableField) &&
        // if adding a manager field to an applicant form, only allow single response, otherwise continue (#1670627)
        (addingManagerFieldToApplicantForm ? field.isSingleResponse : true) &&
        // Only return published for root zone
        (onlyReturnPublished ? field.standardComponentIsPublished : true) &&
        // and finally make sure it isn't on the form already
        !(currentFormRefFieldKeys || []).includes(field.key);
    });
  }

  getCategoryOptionsFromFields (
    fields: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ): TypeaheadSelectOption[] {
    const catIds = uniq(fields.map((field) => field.categoryId));

    return this.arrayHelper.sort(
      catIds.map((catId) => {
        return {
          label: this.categoryNameMap[catId],
          value: catId
        };
      }),
      'label'
    );
  }

  getParentRefFieldOptionsFromDependentCDTGuid (
    guid: string,
    refFieldId: number = null
  ): TypeaheadSelectOption<number>[] {
    // refFieldID is optionally passed in to check a very specific scenario
    // if you are editing a parent picklist and update the CDT to a child of the same CDT
    // you were previously using, this refFieldID will allow us to filter the results

    // default options to empty array for cdt without parentPicklist
    const cdt = this.customDataTablesService.getCDTFromGuid(guid);
    let options: TypeaheadSelectOption<number>[] = [];

    if (cdt?.parentPicklistId) {
      // take incoming CDT and get parent
      const parentCDT = this.customDataTablesService.getCDTFromId(cdt.parentPicklistId);
      // filter ref fields down to those that have matching guid to parent
      const filteredRefFields = this.allReferenceFields.filter((refField) => {
        return refField.customDataTableGuid === parentCDT.guid && refField.referenceFieldId !== refFieldId;
      });
      options = filteredRefFields.map((refField) => {
        return {
          label: refField.name,
          value: refField.referenceFieldId
        };
      });
    }
    const sortedOptions = this.arrayHelper.sort(options, 'label');

    return sortedOptions;
  }

  getFormFieldsByType (
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ) {
    return this.allReferenceFields.filter(field => {
      return field.type === type;
    });
  }

  getFormFieldOptionsByType (
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ): TypeaheadSelectOption<number>[] {
    return this.getFormFieldsByType(type)
      .map(field => {
        return {
          label: field.name,
          value: field.referenceFieldId
        };
      });
  }

  getApplicableAggregateFormFields () {
    const unsortedOptions =  [
      ...this.getFormFieldsByType(ReferenceFieldsUI.ReferenceFieldTypes.Number),
      ...this.getFormFieldsByType(
        ReferenceFieldsUI.ReferenceFieldTypes.Currency
      ),
      ...[
        ...this.getFormFieldsByType(
          ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable
        ),
        ...this.getFormFieldsByType(
          ReferenceFieldsUI.ReferenceFieldTypes.Radio
        )
      ].filter(field => {
        const dataTable = this.customDataTablesService.get('customDataTables')
        .find(table => {
          return table.guid === field.customDataTableGuid;
        });

        return dataTable?.dataType === PicklistDataType.Numeric && !field.supportsMultiple;
      })
    ].filter(field => {
      return field.formAudience === FormAudience.MANAGER &&
        !field.isTableField;
    }).map((field) => {
      return {
        label: field.name,
        value: field.referenceFieldId
      };
    });

    return this.arrayHelper.sort(unsortedOptions, 'label');
  }

  getDuplicateFieldAlert (
    existingFieldId: number,
    formVal: ReferenceFieldAPI.ReferenceFieldBaseModel
  ) {
    if (
      !existingFieldId &&
      formVal.name &&
      formVal.categoryId &&
      formVal.type &&
      formVal.formAudience
    ) {
      const found = this.allReferenceFields.find((field) => {
        const matchesBasics = (field.name.toLowerCase() === formVal.name.toLowerCase()) &&
          (field.categoryId === formVal.categoryId) &&
          (field.type === formVal.type) &&
          (field.formAudience === formVal.formAudience);
        if (!matchesBasics) {
          return false;
        }
        let matchesMultiSettings = true;
        if (
          formVal.type === ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable ||
          formVal.type === ReferenceFieldsUI.ReferenceFieldTypes.TextArea ||
          formVal.type === ReferenceFieldsUI.ReferenceFieldTypes.TextField ||
          formVal.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload
        ) {
          matchesMultiSettings = field.supportsMultiple === formVal.supportsMultiple;
        }
        if (!matchesMultiSettings) {
          return false;
        }
        let matchesDataTable = true;
        if (this.doesTypeHaveOptions(formVal.type)) {
          matchesDataTable = field.customDataTableGuid === formVal.customDataTableGuid;
        }

        return matchesBasics && matchesMultiSettings && matchesDataTable;
      });

      if (found) {
        return this.i18n.translate(
          'FORMS:textSettingsMatchAnExistingField',
          {
            fieldType: this.getFieldTypeTranslatedWithIcon(
              formVal.type
            ).label.toLowerCase()
          },
          'The form field settings you entered match an existing __fieldType__ field.'
        );
      }
    }

    return undefined;
  }

  setCurrentFormRefFieldsInfo (components: FormDefinitionComponent[]) {
    const refFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[] = [];
    this.componentHelper.eachComponent(components, (comp) => {
      if (this.componentHelper.isReferenceFieldComp(comp.type)) {
        const key = this.componentHelper.getRefFieldKeyFromCompType(comp.type);
        const refField = this.getReferenceFieldByKey(key);
        if (refField) {
          refFields.push(refField);
        }
      }
    });
    this.set('currentFormRefFields', refFields);
  }

  addCurrentFormRefFields (components: FormDefinitionComponent[]) {
    const refFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[] = [];
    const newRefFieldComps = components.filter((comp) => this.componentHelper.isReferenceFieldComp(comp.type));
    newRefFieldComps.forEach((comp) => {
      const key = this.componentHelper.getRefFieldKeyFromCompType(comp.type);
        const refField = this.getReferenceFieldByKey(key);
        if (refField) {
          refFields.push(refField);
        }
    });
    this.set('currentFormRefFields',
      [
        ...this.currentFormRefFields,
        ...refFields
      ]
    );
  }

  getAllAggregateFields () {
    return this.allReferenceFields.filter((field) => {
      return !!field.aggregateType;
    });
  }

  adaptFormioChangesForSave (
    changeMap: ReferenceFieldsUI.RefResponseMap,
    applicationFormId: number,
    applicationId: number,
    revisionId: number,
    isNew: boolean /* If new (offline only), we track off revisionId */
    /* because applicationFormId is not defined yet */,
    isManagerForm: boolean
  ): ReferenceFieldsUI.AdaptRefChangesResponse {
    let needToSaveFileIds = false;
    Object.keys(changeMap).forEach((refFieldKey) => {
      const field = this.getReferenceFieldByKey(refFieldKey);
      if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload) {
        needToSaveFileIds = true;
      } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
        const tableColumns = this.tableColumnsMap[field.referenceFieldId];
        needToSaveFileIds = tableColumns?.some((column) => {
          return column.referenceField.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload;
        });
      }
    });
    const nonTableChangeMap: ReferenceFieldsUI.RefResponseMap = {};
    const tableChangeMap: ReferenceFieldsUI.RefResponseMap = {};
    Object.keys(changeMap).forEach((refFieldKey) => {
      const field = this.getReferenceFieldByKey(refFieldKey);
      const isTable = field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset;
      if (isTable) {
        tableChangeMap[refFieldKey] = changeMap[refFieldKey];
      } else {
        nonTableChangeMap[refFieldKey] = changeMap[refFieldKey];
      }
    });
    let standardChangeValues: ReferenceFieldAPI.ApplicationRefFieldResponseForApi[] = [];
    let tableChangeValues: ReferenceFieldAPI.TableChangeResponse = {
      updates: [],
      deletions: []
    };
    if (Object.keys(nonTableChangeMap).length > 0) {
      standardChangeValues = this.mapReferenceFieldResponsesForAPI(nonTableChangeMap, isManagerForm);
    }
    if (Object.keys(tableChangeMap).length > 0) {
      tableChangeValues = this.mapTableResponsesForAPI(
        tableChangeMap,
        applicationFormId,
        applicationId,
        revisionId,
        isNew
      );
    }

    return {
      needToSaveFileIds,
      standardChangeValues,
      tableChangeValues,
      tableChangeMap
    };
  }

  getAudiencesForRefFieldType (
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ): Record<keyof typeof FormAudience, boolean> {
    switch (type) {
      case ReferenceFieldsUI.ReferenceFieldTypes.Table:
        return {
          APPLICANT: true,
          MANAGER: false
        };
      case ReferenceFieldsUI.ReferenceFieldTypes.Aggregate:
        const hasApplicantAggregates = this.allReferenceFields.some((field) => {
          return !!field.aggregateType &&
            field.formAudience === FormAudience.APPLICANT;
        });

        return {
          APPLICANT: hasApplicantAggregates,
          MANAGER: true
        };
      default:
        return {
          APPLICANT: true,
          MANAGER: true
        };
    }
  }

  handleChangeTracking (
    changes: FormioChanges[],
    application: Partial<BaseApplication>
  ): FormioChangeTracker {
    let appNeedsUpdated = false;
    const refChangeTracker: ReferenceFieldsUI.RefResponseMap = {};
    const applicationFieldChanges = changes.filter((change) => {
      return !change.isReferenceField;
    });
    if (applicationFieldChanges.length > 0) {
      appNeedsUpdated = true;
      this.updateApplicationWithNewValues(
        applicationFieldChanges,
        application
      );
    }
    const refFieldChanges = changes.filter((change) => change.isReferenceField);
    if (refFieldChanges.length > 0) {
      refFieldChanges.forEach((change) => {
        const refField = this.getReferenceFieldByKey(change.key);
        const answer = change.value;
        const untrackedField = !!refField.aggregateType;
        // make sure we aren't clearing out answers
        // when a user doesn't have permission to view masked data
        // also make sure we aren't trying to save aggregate fields which should be untracked
        if (
          (!refField.isMasked || answer !== '') &&
          !untrackedField
        ) {
          refChangeTracker[change.key] = answer;
          application.referenceFields[change.key] = answer;
        }
      });
    }

    return {
      appNeedsUpdated,
      refChangeTracker
    };
  }

  /**
   *
   * @param field: the reference field
   * @param component: the form component
   * @param applyDefaultValue: should we apply the default?
   * @returns the blank value for the field type
   */
  getBlankValueForFormField (
    field: ReferenceFieldAPI.ReferenceFieldDisplayModel,
    component: FormDefinitionComponent,
    applyDefaultValue: boolean
  ): FormioAnswerValues {
    if (field) {
      if (
        field.supportsMultiple ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload
      ) {
        const requiresOneItem = [
          ReferenceFieldsUI.ReferenceFieldTypes.TextField,
          ReferenceFieldsUI.ReferenceFieldTypes.TextArea
        ].includes(field.type);
        if (requiresOneItem) {
          return [''];
        }

        return [];
      } else if (
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextField ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.TextArea ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable
      ) {
        if (applyDefaultValue && !!component.defaultVal) {
          component.appliedDefaultVal = true;

          return component.defaultVal;
        }

        return '';
      } else if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Currency) {
        const defaultCurrency = this.clientSettingsService.defaultCurrency;
        const currencyOptions = this.currencyService.getCurrencyOptionsForComponent(
          '',
          component.useCustomCurrency,
          component.customCurrency
        );
        let amountForControl = 0;
        if (applyDefaultValue && !!component.defaultVal) {
          component.appliedDefaultVal = true;

          amountForControl = +component.defaultVal;
        }

        return {
          amountInDefaultCurrency: null,
          amountEquivalent: null,
          amountForControl,
          currency: this.componentHelper.getCurrencyForFormFieldControl(
            null,
            component.useCustomCurrency,
            component.customCurrency,
            currencyOptions,
            defaultCurrency,
            this.userService.get('lastSelectedCurrency')
          )
        };
      }
    }

    if (applyDefaultValue && !!component.defaultVal) {
      component.appliedDefaultVal = true;

      return component.defaultVal;
    }

    return null;
  }

  updateApplicationWithNewValues (
    applicationFieldChanges: FormioChanges[],
    application: Partial<BaseApplication>
  ) {
    applicationFieldChanges.forEach((change) => {
      switch (change.type) {
        case 'amountRequested':
          const details = (change.value as CurrencyValue);
          application.amountRequestedForEdit = details.amountForControl;
          application.currencyRequested = details.currency || application.currencyRequested;
          break;
        case 'inKindItems':
          application.inKindItems = change.value as InKindRequestedItem[] || [];
          break;
        case 'designation':
          application.designation = change.value as string;
          break;
        case 'careOf':
          application.careOf = change.value as string;
          break;
        case 'specialHandling':
          application.specialHandling = change.value as SpecialHandling;
          break;
        case 'decision':
          application.decision = change.value as FormDecisionTypes;
          break;
        case 'reviewerRecommendedFundingAmount':
          const value = (change.value as CurrencyValue);
          application.reviewerRecommendedFundingAmount = value.amountForControl;
          break;
        case 'reportField':
        case 'employeeSSO':
          break;
      }
    });
  }

  adaptAggregateTypeFromImport (
    aggregateType: 'sum'|'min'|'max'|'count'|'average'
  ) {
    const types = ReferenceFieldAPI.ReferenceFieldAggregateType;
    switch (aggregateType) {
      case 'sum':
        return types.Sum;
      case 'min':
        return types.Min;
      case 'max':
        return types.Max;
      case 'count':
        return types.Count;
      case 'average':
        return types.Average;
    }
  }

  getCanUpdateToSingleResponse (referenceFieldId: number) {
    return this.referenceFieldsResources.getCanUpdateRefFieldToSingleResponse(
      referenceFieldId
    );
  }

  convertRefFieldKeysToIds (keys: string[]): number[] {
    return (keys || []).filter((key) => {
      return this.referenceFieldMap[key];
    }).map((key) => {
      return this.referenceFieldMap[key].referenceFieldId;
    });
  }

  /**
   *
   * @param type: reference field type
   * @returns can it be multi?
   */
  canBeMulti (type: ReferenceFieldsUI.ReferenceFieldTypes) {
    return [
      ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable,
      ReferenceFieldsUI.ReferenceFieldTypes.TextArea,
      ReferenceFieldsUI.ReferenceFieldTypes.TextField,
      ReferenceFieldsUI.ReferenceFieldTypes.FileUpload,
      ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes
    ].includes(type);
  }

  /**
   * Picklists are required to be multi, so we don't show the option
   *
   * @param type: reference field type
   * @returns do we show the checkbox?
   */
  showSupportsMultiCheckbox (type: ReferenceFieldsUI.ReferenceFieldTypes) {
    return [
      ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable,
      ReferenceFieldsUI.ReferenceFieldTypes.TextArea,
      ReferenceFieldsUI.ReferenceFieldTypes.TextField,
      ReferenceFieldsUI.ReferenceFieldTypes.FileUpload
    ].includes(type);
  }

  isCorrectTypeForMaskAndEncyrpt (
    type: ReferenceFieldsUI.ReferenceFieldTypes,
    isTableField: boolean
  ) {
    const passesType = [
      ReferenceFieldsUI.ReferenceFieldTypes.Number,
      ReferenceFieldsUI.ReferenceFieldTypes.TextField,
      ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable,
      ReferenceFieldsUI.ReferenceFieldTypes.Radio,
      ReferenceFieldsUI.ReferenceFieldTypes.Date,
      ReferenceFieldsUI.ReferenceFieldTypes.SelectBoxes,
      ReferenceFieldsUI.ReferenceFieldTypes.ExternalAPI
    ].includes(type);

    return passesType && !isTableField;
  }

  canBeTableField (
    type: ReferenceFieldsUI.ReferenceFieldTypes,
    supportsMultiple: boolean,
    formAudience: FormAudience,
    isExistingTableField: boolean
  ) {
    if (isExistingTableField) {
      return true;
    }
    const typePasses = [
      ReferenceFieldsUI.ReferenceFieldTypes.TextField,
      ReferenceFieldsUI.ReferenceFieldTypes.TextArea,
      ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable,
      ReferenceFieldsUI.ReferenceFieldTypes.Number,
      ReferenceFieldsUI.ReferenceFieldTypes.Checkbox,
      ReferenceFieldsUI.ReferenceFieldTypes.Date,
      ReferenceFieldsUI.ReferenceFieldTypes.Radio,
      ReferenceFieldsUI.ReferenceFieldTypes.Currency,
      ReferenceFieldsUI.ReferenceFieldTypes.FileUpload
    ].includes(type);

    return typePasses &&
      !supportsMultiple &&
      formAudience === FormAudience.APPLICANT;
  }

  getAvailableTableFields () {
    return this.allReferenceFields.filter((field) => {
      return field.isTableField &&
        !field.aggregateTableReferenceFieldId;
    });
  }

  getSubsetRowFields (audience: FormAudience) {
    return this.allReferenceFields.filter((field) => {
      return field.type === ReferenceFieldsUI.ReferenceFieldTypes.DataPoint &&
        field.formAudience === audience;
    });
  }

  getTableFieldSelectOptions () {
    return this.allReferenceFields.filter((field) => {
      return field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table &&
        field.referenceFieldTableId;
    }).map((field) => {
      return {
        label: field.name,
        value: field.key
      };
    });
  }

  findAggregateForTableColumn (
    tableRefFieldId: number,
    columnRefFieldId: number
  ) {
    return this.allReferenceFields.find((refField) => {
      return !!refField.aggregateType &&
        refField.aggregateTableReferenceFieldId === tableRefFieldId &&
        refField.parentReferenceFieldId === columnRefFieldId;
    });
  }

  async getTableFields (
    referenceFieldId: number
  ): Promise<ReferenceFieldsUI.TableFieldForUi[]> {
    const refField = this.referenceFieldMapById[referenceFieldId];
    let fields: ReferenceFieldAPI.TableField[] = [];
    if (refField.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
      fields = await this.referenceFieldsResources.getTableFields(referenceFieldId);
    } else if (refField.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
      fields = await this.referenceFieldsResources.getSubsetFields(referenceFieldId);
    }
    const adapted = fields.map<ReferenceFieldsUI.TableFieldForUi>((field) => {
      const summarizeData = !!field.aggregateColumnReferenceFieldId;
      const summarizeLabel = this.referenceFieldMapById[
        field.aggregateColumnReferenceFieldId
      ]?.name ?? '';

      return {
        ...field,
        summarizeData,
        summarizeLabel,
        referenceField: this.referenceFieldMapById[field.referenceFieldId]
      };
    });

    return this.arrayHelper.sort(adapted, 'columnOrder');
  }
  /**
   *
   * @param formCount number of forms this component is used on
   * @param usedOnReports boolean true means field is used on reports
   * @param referenceFieldId used to check for parent/child field relationship. cannot remove if parent
   * @param standardComponentIsPublished true if published standard product field, we don't support removing these
   * @param isRootZone if true we allow removing standard product fields that are not published
   * @returns string if unable to remove, void if able to remove
   */
  getTooltipForDeleteRefField (
    formCount: number,
    usedOnReports: boolean,
    referenceFieldId: number,
    standardComponentIsPublished: boolean,
    isRootZone: boolean
  ) {
    if (isRootZone && standardComponentIsPublished) {
      return this.i18n.translate(
        'FORMS:textCannotDeletePublishedRootZoneField',
        {},
        'This field cannot be removed because it is published'
      );
    } else if (standardComponentIsPublished) {
      return this.i18n.translate(
        'FORMS:textCannotDeleteStandardProductField',
        {},
        'Standard product fields cannot be removed'
      );
    }
    if (formCount > 0) {
      return this.i18n.translate(
        'FORMS:textCannotRemoveReferenceField',
        {
          count: formCount
        },
        'This field cannot be removed, it is currently in use on __count__ form(s)'
      );
    } else if (usedOnReports) {
      return this.i18n.translate(
        'FORMS:textCannotRemoveReferenceFieldOnReport',
        'This field cannot be removed, it is currently in use on at least one report'
      );
    } else if (this.isParentRefFieldMap[referenceFieldId]) {
      return this.i18n.translate(
        'FORMS:textCannotRemoveReferenceFieldParent',
        {},
        'This field cannot be removed because it relates to another field'
      );
    }

    return '';
  }

  /* Returns the tooltip if they are not allowed to :
   1. Uncheck the 'Summarize Data' box on a table column
   2. Or remove a column that is summarizing data */
  getDisableSummarizeDataTooltip (
    summarizeData: boolean,
    tableRefId: number,
    columnRefId: number,
    forDeletingTableColumn = false
  ): string {
    /* Disable the ability to remove an aggregate from table column */
    /* If the aggregate has been used on forms or reports */
    if (summarizeData && tableRefId && columnRefId) {
      const existingAggregate = this.findAggregateForTableColumn(
        tableRefId,
        columnRefId
      );
      if (existingAggregate?.formCount > 0) {
        return this.i18n.translate(
          forDeletingTableColumn ?
            'FORMS:textCannotDeleteTableColumnOnForms' :
            'FORMS:textCannotUncheckSummaryOnForms2',
          {
            count: existingAggregate.formCount
          },
          forDeletingTableColumn ?
            `Cannot remove this table column because it's summarized data is used on __count__ form(s)` :
            `Cannot be unchecked because the summarized data is used on __count__ form(s)`
        );
      } else if (existingAggregate?.usedOnReports) {
        return this.i18n.translate(
          forDeletingTableColumn ?
            'FORMS:textCannotDeleteTableColumnOnReports' :
            'FORMS:textCannotUncheckSummaryOnReports',
          {},
          forDeletingTableColumn ?
            `Cannot remove this table column because it's summarized data is used on at least one report` :
            'Cannot be unchecked because summarized data is used on at least one report'
        );
      }
    }

    return '';
  }

  setAllTableAndSubsetColumnsOnForm (tableRefIds: number[]) {
    tableRefIds = uniq(tableRefIds);

    return Promise.all(tableRefIds.map((id) => {
      const field = this.referenceFieldMapById[id];
      if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
        return this.setColumnsForTable(id);
      } else {
        return this.setDataPointsForSubset(id);
      }
    }));
  }


  async setDataPointsForSubset (subsetId: number) {
    if (!this.dataPointsMap[subsetId]) {
      const fields = await this.getSubsetRows(subsetId);

      this.set('dataPointsMap', {
        ...this.dataPointsMap,
        [subsetId]: this.arrayHelper.sort(fields, 'columnOrder')
      });
    }

    return this.dataPointsMap[subsetId];
  }

  async setColumnsForTable (tableRefId: number) {
    if (!this.tableColumnsMap[tableRefId]) {
      const fields = await this.getTableFields(tableRefId);

      this.set('tableColumnsMap', {
        ...this.tableColumnsMap,
        [tableRefId]: fields
      });
    }

    return this.tableColumnsMap[tableRefId];
  }

  getTableFormKey (
    appFormIdOrRevisionId: number,
    tableReferenceFieldId: number
  ) {
    return `${appFormIdOrRevisionId}_${tableReferenceFieldId}`;
  }

  setApplicationFormTableRowsMap (
    appFormIdOrRevisionId: number,
    tableResponses: ReferenceFieldsUI.RefResponseMap
  ) {
    const additionsToMap = Object.keys(tableResponses).reduce((acc, key) => {
      const field = this.referenceFieldMap[key];
      const response = tableResponses[key] as ReferenceFieldsUI.TableResponseRowForUi[];

      return {
        ...acc,
        [this.getTableFormKey(appFormIdOrRevisionId, field.referenceFieldId)]: response
      };
    }, {});
    const updatedMap: Record<string, ReferenceFieldsUI.TableResponseRowForUi[]> = {
      ...this.applicationFormTableRowsMap,
      ...additionsToMap
    };

    this.set('applicationFormTableRowsMap', updatedMap);
  }

  updateApplicationFormTableRowsMapAfterCopy (
    oldApplicationFormId: number,
    newApplicationFormId: number,
    tableIds: number[]
  ) {
    tableIds.forEach((tableId) => {
      const oldKey = this.getTableFormKey(oldApplicationFormId, tableId);
      const newKey = this.getTableFormKey(newApplicationFormId, tableId);
      const oldValue = this.applicationFormTableRowsMap[oldKey];
      if (oldValue) {
        const updatedMap: Record<string, ReferenceFieldsUI.TableResponseRowForUi[]> = {
          ...this.applicationFormTableRowsMap,
          [oldKey]: undefined,
          [newKey]: oldValue.map((value) => {
            return {
              ...value,
              rowId: null
            };
          })
        };
        this.set('applicationFormTableRowsMap', updatedMap);
      }
    });
  }

  returnTableRowImportData (
    tableId: number,
    tableRows: ReferenceFieldsUI.TableResponseRowForUiMapped[],
    download: boolean,
    downloadFormat: TableDataDownloadFormat
  ) {
    let columns: ReferenceFieldsUI.TableFieldForUi[];
    const refField = this.referenceFieldMapById[tableId];
    let objectsToDownload: Record<string, FormioAnswerValues>[] = [];
    if (refField.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
      columns = this.tableColumnsMap[tableId];
      objectsToDownload = tableRows.map((row) => {
        return columns.reduce((acc, column) => {
          return {
            ...acc,
            [column.label]: row.responses[column.referenceField.key]
          };
        }, {});
      });
    } else if (refField.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
      columns = this.dataPointsMap[tableId];
      const row = tableRows[0];
      if (row) {
        const collectionTypeMap = this.getCollectionTypeMap();
        columns.forEach((column) => {
          objectsToDownload.push({
            [refField.name]: column.label,
            [collectionTypeMap[refField.subsetCollectionType]]: row.responses[column.referenceField.key]
          });
        });
      }

    }
    if (objectsToDownload.length > 0) {
      const csv = this.fileService.convertObjectArrayToCSVString(objectsToDownload);
      if (download) {
        this.fileService.downloadByFormat(csv, downloadFormat);
      }

      return csv;
    } else {
      const cdtItemsMap = this.customDataTablesService.getCdtItemsMapForRow(
        columns
      );
      const ValidationClass = this.getValidationClassForTableRowImport(
        tableId,
        cdtItemsMap
      );
      const csv = this.dynamicCsvService.getSample(ValidationClass);
      if (download) {
        this.fileService.downloadByFormat(csv, downloadFormat);
      }

      return csv;
    }
  }

  getValidationClassForTableRowImport (
    tableId: number,
    cdtItemsMap: Record<number, TypeaheadSelectOption[]>
  ) {
    const field = this.referenceFieldMapById[tableId];
    let columns = this.tableColumnsMap[tableId];
    if (field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
      columns = this.dataPointsMap[tableId];
    }
    const ValidatorClass = class { };
    columns.forEach(column => {
      if (column.isRequired) {
        if (
          column.referenceField.type === ReferenceFieldsUI.ReferenceFieldTypes.Checkbox
        ) {
          /* Formio forces checkbox to be "true" if required
          so we need a different required validator */
          RequiredFormCheckbox()(ValidatorClass.prototype, column.label);
        } else {
          Required()(ValidatorClass.prototype, column.label);
        }
      }

      switch (column.referenceField.type) {
        case ReferenceFieldsUI.ReferenceFieldTypes.Checkbox:
          CSVBoolean()(ValidatorClass.prototype, column.label);
          break;
        case ReferenceFieldsUI.ReferenceFieldTypes.Date:
          CSVDate()(ValidatorClass.prototype, column.label);
          break;
        case ReferenceFieldsUI.ReferenceFieldTypes.Number:
          IsNumber({
            evaluateAsString: true
          })(ValidatorClass.prototype, column.label);
          break;
        case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
        case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
          const cdtItems = cdtItemsMap[column.referenceField.referenceFieldId];
          const mappedItems = (cdtItems || []).map((item) => {
            return item.label;
          });
          IsOneOf(mappedItems)(ValidatorClass.prototype, column.label);
          break;
        default:
          IsString()(ValidatorClass.prototype, column.label);
          break;
      }
    });

    return ValidatorClass;
  }

  adaptTableFieldsToFile (
    tableId: number,
    csvRows: Record<string, any>[],
    cdtItemsMap: Record<number, TypeaheadSelectOption[]>
  ) {
    const adapted: {
      key: string;
      value: any;
      applicationReferenceFieldTableRow: number; // index
    }[] = [];
    const columns = this.tableColumnsMap[tableId];
    csvRows.forEach((row, index) => {
      Object.keys(row).forEach((label) => {
        const foundColumn = columns.find((column) => {
          return column.label === label;
        });
        const field = foundColumn.referenceField;
        let value = row[label];
        switch (field.type) {
          case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
          case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
            value = cdtItemsMap[foundColumn.referenceFieldId].find((items) => {
              return items.label === value;
            })?.value;
            break;
        }
        adapted.push({
          key: foundColumn.referenceField.key,
          value,
          applicationReferenceFieldTableRow: index
        });
      });
    });
    const csvString = this.fileService.convertObjectArrayToCSVString(adapted);

    return new Blob([csvString], { type: 'text/csv' });
  }

  async handleImportTableRows (
    applicationId: number,
    applicationFormId: number,
    tableId: number,
    formId: number,
    csvRows: Record<string, any>[],
    cdtItemsMap: Record<number, TypeaheadSelectOption[]>
  ) {
    let passed = false;
    try {
      const file = this.adaptTableFieldsToFile(
        tableId,
        csvRows,
        cdtItemsMap
      );
      await this.referenceFieldsResources.importTableRows(
        applicationId,
        tableId,
        file,
        formId
      );
      this.notifier.success(this.i18n.translate(
        'CONFIG:textSuccessfullyImportedTheFile',
        {},
        'Successfully imported the file'
      ));
      passed = true;
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'CONFIG:textErrorImportingTheFile',
        {},
        'There was an error importing the file'
      ));

      passed = false;
    }

    if (passed) {
      await this.getReferenceFieldResponses(
        applicationId,
        applicationFormId,
        [],
        [tableId],
        null,
        false,
        true
      );
    }

    return passed;
  }

  mapRowsForTable (
    rows: ReferenceFieldsUI.TableResponseRowForUi[],
    tableReferenceFieldId: number,
    formatResponses = false,
    skipCurrencyFormatting = false
  ) {
    const isSubset = this.referenceFieldMapById[tableReferenceFieldId].type === ReferenceFieldsUI.ReferenceFieldTypes.Subset;
    const columns = isSubset ?
      this.dataPointsMap[tableReferenceFieldId] :
      this.tableColumnsMap[tableReferenceFieldId];

    return rows.map<ReferenceFieldsUI.TableResponseRowForUiMapped>((row) => {
      const columnValueMap: Record<string, FormioAnswerValues> = {};
      columns.forEach((column) => {
        columnValueMap[column.referenceField.key] = this.getTableColumnResponse(
          row,
          column,
          formatResponses,
          skipCurrencyFormatting
        );
      });

      return {
        ...row,
        responses: columnValueMap
      };
    });
  }

  mapRowsForSubset (
    dataRows: ReferenceFieldsUI.TableResponseRowForUi[],
    refId: number,
    defaultTo0: boolean
  ): {
    rowsForTable: ReferenceFieldsUI.DataPointForUI[];
    dataRows: ReferenceFieldsUI.TableResponseRowForUi[];
    dataRowsWereUpdated: boolean;
  } {
    const dataPoints = this.dataPointsMap[refId];
    const rowsForTable: ReferenceFieldsUI.DataPointForUI[] = [];
    let needToResetDataRows = false;

    dataPoints.forEach((dataPoint) => {
      let foundInRows = false;
      dataRows.forEach((row) => {
        row.columns.forEach((column) => {
          if (column.referenceFieldId === dataPoint.referenceFieldId) {
            foundInRows = true;
            rowsForTable.push({
              ...dataPoint,
              value: column.value
            });
          }
        });
      });
      // If this is a new subset, the data won't exist yet, so we need to add a blank record
      if (!foundInRows) {
        needToResetDataRows = true;
        rowsForTable.push({
          ...dataPoint,
          value: defaultTo0 ? 0 : null
        });
      }
    });

    // Align dataRows with rowsForTable
    if (needToResetDataRows) {
      const rowId = dataRows[0]?.rowId ?? null;
      dataRows = [{
        columns: rowsForTable.map<ReferenceFieldAPI.ApplicationRefFieldResponse>((rowForTable) => {
          return {
            referenceFieldId: rowForTable.referenceFieldId,
            referenceFieldKey: rowForTable.referenceField.key,
            value: rowForTable.value,
            dateValue: null,
            currencyValue: null,
            numericValue: null,
            file: null,
            files: [],
            applicationFormId: null,
            applicationId: null
          };
        }),
        rowId
      }];
    }

    return {
      rowsForTable,
      dataRows,
      dataRowsWereUpdated: needToResetDataRows
    };
  }

  getTableColumnResponse (
    row: ReferenceFieldsUI.TableResponseRowForUi,
    column: ReferenceFieldsUI.TableFieldForUi,
    formatResponses = false,
    skipCurrencyFormatting = false
  ) {
    const response = row.columns.find((col) => {
      return col.referenceFieldId === column.referenceFieldId;
    });
    const value = response?.value ?? null;

    if (!formatResponses) {
      return value;
    }

    return this.formatTableResponse(
      row,
      column,
      value,
      skipCurrencyFormatting
    );
  }

  formatTableResponse (
    row: ReferenceFieldsUI.TableResponseRowForUiMapped|ReferenceFieldsUI.TableResponseRowForUi,
    column: ReferenceFieldsUI.TableFieldForUi,
    value: FormioAnswerValues,
    skipCurrencyFormatting = false
  ) {
    switch (column.referenceField.type) {
      case ReferenceFieldsUI.ReferenceFieldTypes.Date:
        if (value) {
          return this.timezoneService.returnMidnightUTCDateShort(value.toString());
        }

        return value;
      case ReferenceFieldsUI.ReferenceFieldTypes.Number:
        return this.decimal.transform(value as number);
      case ReferenceFieldsUI.ReferenceFieldTypes.Currency:
        if (skipCurrencyFormatting) {
          return (value as CurrencyValue).amountForControl;
        }

        return this.currencyService.formatMoney((value as CurrencyValue).amountForControl);
      case ReferenceFieldsUI.ReferenceFieldTypes.CustomDataTable:
      case ReferenceFieldsUI.ReferenceFieldTypes.Radio:
        const cdtItemsMap = this.customDataTablesService.getCdtItemsMapForRow(
          [column],
          row,
          true
        );
        if (value) {
          return cdtItemsMap[column.referenceFieldId]?.find((option) => {
            return option.value === value;
          })?.label ?? value;
        }

        return value;
      case ReferenceFieldsUI.ReferenceFieldTypes.FileUpload:
        // should always be an array
        if (!(value instanceof Array)) {
          return [];
        }

        return (value as YcFile<File>[]).map((val) => {
          return val.fileUrl;
        });
      default:
        return value;
    }
  }

  getExistingFields (
    rows: ReferenceFieldsUI.TableFieldForUi[],
    availableFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ) {
    const usedIds = rows.filter((row) => {
      return !!row.referenceFieldId;
    }).map((row) => row.referenceFieldId);
    const unusedFields = availableFields.filter((field) => {
      return !usedIds.includes(field.referenceFieldId);
    });

    return unusedFields.length > 0;
  }

  getAvailableFieldsMap (
    rows: ReferenceFieldsUI.TableFieldForUi[],
    availableFields: ReferenceFieldAPI.ReferenceFieldDisplayModel[]
  ) {
    return rows.reduce((acc, _, index) => {
      /* Determine which table/subset field ids are already used */
      const usedIds = rows.filter((_row, _index) => {
        return !!_row.referenceFieldId &&
          index !== _index;
      }).map((_row) => _row.referenceFieldId);
      const availableFieldsSorted = this.arrayHelper.sort(
          availableFields.filter((field) => {
          return !usedIds.includes(field.referenceFieldId);
        }).map((field) => {
          return {
            label: field.name,
            value: field.referenceFieldId
          };
        }),
        'label'
      );

      return {
        ...acc,
        [index]: availableFieldsSorted
      };
    }, {});
  }

  async getSubsetRows (refFieldId: number): Promise<ReferenceFieldsUI.DataPointForUI[]> {
    try {
      const rows = await this.getTableFields(refFieldId);
      const adaptedRows = rows.map((row) => {
        return {
          ...row,
          value: null
        };
      });

      return adaptedRows;
    } catch(e) {
      this.logger.error(e);
      this.notifier.error(
        this.i18n.translate(
          'common:textErrorLoadingFieldGroupOptions',
          {},
          'There was an error loading field group options'
        )
      );

      return null;
    }
  }

  getVisibleTableColumns (
    referenceFieldId: number,
    translations?: Record<string, string>
  ) {
    const field = this.referenceFieldMapById[referenceFieldId];
    let columns;
    if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
      columns = this.tableColumnsMap[referenceFieldId].filter((col) => {
        return col.showInTable;
      });
    } else if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
      columns = this.dataPointsMap[referenceFieldId];
    };

    if (columns?.length > 0) {
      return columns.map((col) => {
        if (translations) {
          col.label = translations[col.label] || col.label;
        }

        return col;
      });
    }

    return [];
  }

  /* If the response is no longer on the form, remove it from responses map */
  filterOutResponsesNoLongerOnForm (
    formDefinition: FormDefinitionForUi[],
    responses: ReferenceFieldsUI.RefResponseMapForAdapting
  ) {
    const refKeysOnForm: string[] = [];
    formDefinition.forEach((tab) => {
      this.componentHelper.eachComponent(tab.components, (component) => {
        const key = this.componentHelper.getRefFieldKeyFromCompType(component.type);
        if (key) {
          refKeysOnForm.push(key);
        }
      });
    });

    Object.keys(responses).forEach((key) => {
      if (!refKeysOnForm.includes(key)) {
        delete responses[key];
      }
    });
  }

  clearOutTableRowIdsForCopy (
    referenceFields: ReferenceFieldsUI.RefResponseMap
  ) {
    /* Clear out rowId for Tables because we need to create new rows on copy */
    Object.keys(referenceFields).forEach((key) => {
      const field = this.referenceFieldMap[key];
      if (
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.Table ||
        field.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset
      ) {
        const response = referenceFields[key] as
          ReferenceFieldsUI.TableResponseRowForUi[];
        response.forEach((tableRow) => {
          tableRow.rowId = null;
        });
      }
    });
  }

  /**
   * On Copy app, we need to clear out any answers that are no longer active on CDTs
   *
   * @param responses: Refrence field responses map
   */
  clearOutInactiveCdtResponses (
    responses: ReferenceFieldsUI.RefResponseMap
  ) {
    Object.keys(responses).forEach((key) => {
      const field = this.referenceFieldMap[key];
      if (field.customDataTableGuid) {
        const response = responses[key];
        if (!!response) {
          const options = this.dataTables[field.customDataTableGuid];
          const inactiveKeys = options.filter((opt) => {
            return !opt.inUse;
          }).map((opt) => {
            return opt.key;
          });
          if (
            response instanceof Array &&
            (response.length > 0)
          ) {
            responses[key] = (responses[key] as string[]).filter((res) => {
              return !inactiveKeys.includes(res);
            });
          } else {
            if (inactiveKeys.includes(response as string)) {
              responses[key] = '';
            }
          }
        }
      }
    });
  }

  /**
   *
   * @param refChangeTracker Reference field response map of changes to check for files against
   * @param newApplicationId new application id
   * @param newApplicationFormId new application form id
   * @param referenceFieldsMap all reference field responses on the form
   * @returns updated change tracker and reference fields map
   */
  async reuploadReferenceFilesOnCopy (
    refChangeTracker: ReferenceFieldsUI.RefResponseMap,
    newApplicationId: number,
    newApplicationFormId: number,
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap
  ): Promise<{
    refChangeTracker: ReferenceFieldsUI.RefResponseMap;
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap;
  }> {
    for (const key in refChangeTracker) {
      if (Object.prototype.hasOwnProperty.call(refChangeTracker, key)) {
        const type = this.referenceFieldMap[key]?.type;
        const isTable = type === ReferenceFieldsUI.ReferenceFieldTypes.Table;
        const isFileUpload = type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload;
        // Find table type file uploads
        if (isTable) {
          const hasFileColumns = this.tableColumnsMap[
            this.referenceFieldMap[key]?.referenceFieldId
          ]?.some((column) => {
            return column.referenceField.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload;
          });
          if (hasFileColumns) {
            const rows = refChangeTracker[key] as ReferenceFieldsUI.TableResponseRowForUi[];
            // Loop over the responses and find any file uploads
            let rowIndex = 0;
            for (const row of rows) {
              let columnIndex = 0;
              for (const column of row.columns) {
                const refType = this.referenceFieldMapById[column.referenceFieldId].type;
                if (refType === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload) {
                  const filesToCopy = column.value as YcFile<File>[] || [];

                  // Get files to upload
                  const filesToUpload = await this.getFilesToReUpload(
                    column.referenceFieldKey,
                    filesToCopy,
                    newApplicationId,
                    newApplicationFormId
                  );
                  // Upload the files and the maps
                  column.value = await this.doUploadOfReferenceFieldTableFiles(
                    filesToUpload,
                    newApplicationId,
                    newApplicationFormId,
                    column
                  );
                  this.updateReferenceFieldsMapWithNewColumnValue(
                    referenceFieldsMap,
                    key,
                    rowIndex,
                    columnIndex,
                    column.value
                  );
                }
                ++columnIndex;
              }
              ++rowIndex;
            }
          }
        } else if (isFileUpload) {
          const filesToCopy = refChangeTracker[key] as YcFile<File>[] || [];
          // Get the files to Upload
          const filesToUpload = await this.getFilesToReUpload(
            key,
            filesToCopy,
            newApplicationId,
            newApplicationFormId
          );
          // Upload the files and the maps
          await this.doUploadOfReferenceFieldFiles(
            filesToUpload,
            newApplicationId,
            newApplicationFormId,
            refChangeTracker,
            referenceFieldsMap
          );
        }
      }
    }

    return {
      refChangeTracker,
      referenceFieldsMap
    };
  }

  /**
   *
   * @param referenceFieldsMap map of answers to update
   * @param tableKey table reference field key
   * @param rowIndex row index
   * @param columnIndex column index
   * @param columnValue column value we want to update map with
   */
  updateReferenceFieldsMapWithNewColumnValue (
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap,
    tableKey: string,
    rowIndex: number,
    columnIndex: number,
    columnValue: FormioAnswerValues
  ) {
    const rows = (referenceFieldsMap[tableKey] as ReferenceFieldsUI.TableResponseRowForUi[]);
    if (rows && rows[rowIndex]) {
      rows[rowIndex].columns[columnIndex].value = columnValue;
    }
  }

  /**
   *
   * @param files the files to check if re-upload is needed
   * @param currentApplicationId the current application id
   * @param currentApplicationFormId the current application form id
   * @returns boolean - true if we need to re-upload these files for the current application
   */
  getHasFilesToReUpload (
    files: YcFile<File>[],
    currentApplicationId: number,
    currentApplicationFormId: number
  ): boolean {
    return files.some((file) => {
      const {
        applicationFormId,
        applicationId
      } = this.applicationFileService.breakDownloadUrlDownToObject(file.fileUrl);

      return this.getDoesFileNeedReUploaded(
        +applicationId,
        +applicationFormId,
        currentApplicationId,
        currentApplicationFormId
      );
    });
  }

  /**
   *
   * @param fileApplicationId application id of file
   * @param fileApplicationFormId application form id of file
   * @param currentApplicationId current application id
   * @param currentApplicationFormId current application form id
   * @returns boolean - true if the file needs re-uploaded
   */
  getDoesFileNeedReUploaded (
    fileApplicationId: number,
    fileApplicationFormId: number,
    currentApplicationId: number,
    currentApplicationFormId: number
  ): boolean {
    return (+fileApplicationFormId !== +currentApplicationFormId) ||
      (+fileApplicationId !== +currentApplicationId);
  }

  /**
   *
   * @param key reference field key
   * @param files files for this key to check against for re-upload
   * @param currentApplicationId current application id
   * @param currentApplicationFormId current application form id
   * @returns converts the fileUrl to a File and returns an array of the ones that need re-uploaded
   */
  async getFilesToReUpload (
    key: string,
    files: YcFile<File>[],
    currentApplicationId: number,
    currentApplicationFormId: number
  ): Promise<ReferenceFieldsUI.FileNeedingUploaded[]> {
    const filesToUpload: ReferenceFieldsUI.FileNeedingUploaded[] = [];
    const hasFilesToReUpload = this.getHasFilesToReUpload(
      files,
      currentApplicationId,
      currentApplicationFormId
    );
    if (hasFilesToReUpload) {
      for (const file of files) {
        const {
          applicationFormId,
          applicationId,
          fileId
        } = this.applicationFileService.breakDownloadUrlDownToObject(file.fileUrl);
        const fileNeedsReUploaded = this.getDoesFileNeedReUploaded(
          +applicationId,
          +applicationFormId,
          currentApplicationId,
          currentApplicationFormId
        );
        if (fileNeedsReUploaded) {
          const blob = await this.applicationFileService.getFileFromFileInfo(
            +applicationFormId,
            +applicationId,
            +fileId
          );
          const newFile = this.fileService.convertBlobToFile(
            blob,
            file.fileName
          );
          file.file = newFile;
          filesToUpload.push({
            key,
            referenceFieldId: this.referenceFieldMap[key].referenceFieldId,
            file
          });
        }
      }
    }

    return filesToUpload;
  }

  /**
   * After copying an application, this ensures that all files are re-uploaded to the new application
   *
   * @param applicationId application id
   * @param applicationFormId application form id
   * @param refChangeTracker map of changes by key
   * @param referenceFields map of all reference field answers by key
   * @returns the updated change tracker and reference fields map
   */
  async ensureFormFieldFilesAreUploadedForCopy (
    applicationId: number,
    applicationFormId: number,
    refChangeTracker: ReferenceFieldsUI.RefResponseMap,
    referenceFields: ReferenceFieldsUI.RefResponseMap
  ): Promise<{
    refChangeTracker: ReferenceFieldsUI.RefResponseMap;
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap;
  }> {
    return this.reuploadReferenceFilesOnCopy(
      refChangeTracker,
      applicationId,
      applicationFormId,
      referenceFields
    );
  }

  canUpdateCdtOnField (
    referenceFieldId: number,
    type: ReferenceFieldsUI.ReferenceFieldTypes
  ) {
    if (this.doesTypeHaveOptions(type)) {
      return this.referenceFieldsResources.canUpdateCdtOnField(
        referenceFieldId
      );
    }

    return false;
  };

  convertCheckboxValueToNumber (value: boolean): number {
    if (value === true) {
      return 1;
    } else if (value === false) {
      return 0;
    } else {
      return null;
    };
  }

  convertNumberToCheckboxValue (value: number): boolean {
    if (value === 1) {
      return true;
    } else if (value === 0) {
      return false;
    } else {
      return false;
    };
  }

  adaptDataSetResponse (
    dataPoints: ReferenceFieldsUI.DataPointForUI[]
  ): ReferenceFieldAPI.ApplicationRefFieldResponse[] {
    return dataPoints.map((dataPoint) => {
      const dataPointValue = dataPoint.value;
      const dataPointRefFieldID = dataPoint.referenceFieldId;
      const dataPointRefField = this.referenceFieldMapById[
        dataPointRefFieldID
      ];

      return {
        currencyValue: null,
        dateValue: null,
        numericValue: null,
        value: dataPointValue,
        referenceFieldId: +dataPointRefFieldID,
        referenceFieldKey: dataPointRefField.key,
        file: null,
        files: [],
        applicationFormId: null,
        applicationId: null
      };
    });
  }

  getFieldsAvailableToMergeWithStandard (
    standardRefFieldId: number
  ): ReferenceFieldAPI.ReferenceFieldDisplayModel[] {
    const standardField = this.referenceFieldMapById[standardRefFieldId];

    return this.allReferenceFields.filter((field) => {
      return !field.isStandardProductField &&
        this.areAttributesTheSame(
          [
            standardField,
            field
          ],
          [
            'type',
            'formAudience',
            'customDataTableGuid',
            'supportsMultiple',
            'parentReferenceFieldId',
            'isSingleResponse',
            'isEncrypted',
            'isMasked',
            'isTableField',
            'formatType'
          ]
        );
    });
  }

  getCategoryDropdownFilter () {
    return new TopLevelFilter(
      'checkboxDropdown',
      'categoryId',
      [],
      this.i18n.translate(
        'GLOBAL:textSearchByCategory',
        {},
        'Search by category'
      ),
      {
        selectOptions: this.arrayHelper.sort([
          {
            label: this.i18n.translate(
              'common:textOther',
              {},
              'Other'
            ),
            value: 0,
            filterTypeOverride: 'ib'
          },
          ...this.categoryOptions
        ], 'label'),
        filterObjectName: this.i18n.translate(
          'GLOBAL:textCategory', {}
        ).toLowerCase(),
        filterObjectNamePlural: this.i18n.translate(
          'GLOBAL:textCategories', {}
        ).toLowerCase()
      },
      undefined,
      undefined,
      true
    );
  }

  getFilterForNameOrKey () {
    return new TopLevelFilter(
      'text',
      'nameOrKey',
      '',
      this.i18n.translate(
        'common:textSearchByNameOrKey',
        {},
        'Search by name or key'
      ),
      undefined,
      undefined,
      [{
        column: 'name',
        filterType: 'cn'
      }, {
        column: 'key',
        filterType: 'cn'
      }]
    );
  }

  areAttributesTheSame (
    fields: ReferenceFieldAPI.ReferenceFieldDisplayModel[],
    attrs: (keyof ReferenceFieldAPI.ReferenceFieldDisplayModel)[]
  ) {
    const field1 = fields[0];
    const field2 = fields[1];

    return attrs.every((attr) => {
      return field1[attr] === field2[attr];
    });
  }

  async checkMergeForConflicts (
    standardProductReferenceFieldId: number,
    idsToMerge: number[]
  ): Promise<{
    hasConflicts: boolean;
    hasOtherError: boolean;
  }> {
    try {
      await this.referenceFieldsResources.checkMergeForConflicts(
        standardProductReferenceFieldId,
        idsToMerge
      );

      return {
        hasConflicts: false,
        hasOtherError: false
      };
    } catch (err) {
      const e = err as HttpErrorResponse;
      this.logger.error(e);
      if (
        e?.error?.message === 'There are data conflicts among these reference fields. One or more reference fields exist on an application. Merge would result in data loss.'
      ) {
        this.notifier.error(this.i18n.translate(
          'common:textCantMergeWithStandardConflict2',
          {},
          `These fields both contain information related to the same application(s) so cannot be merged.`
        ));

        return {
          hasConflicts: true,
          hasOtherError: false
        };
      } else {
        this.notifier.error(this.i18n.translate(
          'common:textErrorMergeStandardProductField',
          {},
          'There was an error merging with the standard product field'
        ));

        return {
          hasConflicts: false,
          hasOtherError: true
        };
      }
    }
  }

  async handleMergeWithStandardField (
    standardFieldId: number,
    fieldIdsToMerge: number[]
  ) {
    try {
      await this.referenceFieldsResources.mergeWithStandardProductField(
        standardFieldId,
        fieldIdsToMerge
      );
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'common:textSuccessMergeStandardProductField',
        {},
        'Successfully merged with the standard product field'
      ));
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'common:textErrorMergeStandardProductField',
        {},
        'There was an error merging with the standard product field'
      ));
    }
  }

  handleDependentPicklistValidityForQuickAdd (
    componentsToAdd: ReferenceFieldsUI.QuickAddField[],
    invalidDependentPicklists: ReferenceFieldsUI.InvalidDependentPicklist[],
    fieldToAdd?: ReferenceFieldsUI.QuickAddField,
    fieldToRemove?: ReferenceFieldsUI.QuickAddField
  ) {
    if (fieldToAdd && ('referenceFieldId' in fieldToAdd)) {
      const isParentPicklist = this.getIsParentRefField(
        fieldToAdd.referenceFieldId
      );
      const requiresParent = !!fieldToAdd.parentReferenceFieldId &&
        !fieldToAdd.aggregateType;
      if (requiresParent) {
        const result = this.checkForParentPicklistOnForm(
          fieldToAdd.parentReferenceFieldId
        );
        if (!result.parentIsOnForm) {
          // If parent not on form, see if they are in the componentsToAdd array
          const foundParent = componentsToAdd.find((comp) => {
            return ('referenceFieldId' in comp) &&
              comp.referenceFieldId === result.parentPicklist.referenceFieldId;
          });
          if (!foundParent) {
            invalidDependentPicklists.push({
              fieldWithoutParent: fieldToAdd,
              parentPicklist: result.parentPicklist
            });
          }
        }
      } else if (isParentPicklist) {
        const child = this.getChildOfParentRefField(
          fieldToAdd.referenceFieldId
        );
        // If this child was found in the invalid array, remove it
        const childIndex = invalidDependentPicklists.findIndex((item) => {
          return item.fieldWithoutParent.referenceFieldId === child.referenceFieldId;
        });
        if (childIndex > -1) {
          invalidDependentPicklists = [
            ...invalidDependentPicklists.slice(0, childIndex),
            ...invalidDependentPicklists.slice(childIndex + 1)
          ];
        }
      }
    } else if (fieldToRemove && ('referenceFieldId' in fieldToRemove)) {
      // Check to see if the field we are removing had an error.
      // If so error no longer relevant
      const fieldIndex = invalidDependentPicklists.findIndex((item) => {
        return item.fieldWithoutParent.referenceFieldId === fieldToRemove.referenceFieldId;
      });
      if (fieldIndex !== -1) {
        invalidDependentPicklists = [
          ...invalidDependentPicklists.slice(0, fieldIndex),
          ...invalidDependentPicklists.slice(fieldIndex + 1)
        ];
      }
      // If we remove a parent, we need to add validation that the child is not being added
      const child = this.getChildOfParentRefField(
        fieldToRemove.referenceFieldId
      );
      if (child) {
        const childIsBeingAdded = componentsToAdd.find((comp) => {
          return ('referenceFieldId' in comp) &&
            comp.referenceFieldId === child.referenceFieldId;
        });
        if (childIsBeingAdded) {
          this.handleDependentPicklistValidityForQuickAdd(
            componentsToAdd,
            invalidDependentPicklists,
            child
          );
        }
      }
    }

    return invalidDependentPicklists;
  }

  async handleTableFileUploads (
    tableColumns: ReferenceFieldsUI.TableFieldForCrudUi[],
    responseMap: ReferenceFieldsUI.RefResponseMap,
    applicationId: number,
    applicationFormId: number
  ): Promise<ReferenceFieldsUI.RefResponseMap> {
    try {
      const filesNeedingUploaded = this.getTableFilesNeedingUploaded(
        tableColumns,
        responseMap
      );

      const response = await this.doUploadOfReferenceFieldFiles(
        filesNeedingUploaded,
        applicationId,
        applicationFormId,
        responseMap,
        {}
      );

      return response.refChangeTracker;
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(this.i18n.translate(
        'common:textThereWasAnErrorSaving',
        {},
        'There was an error saving'
      ));

      return null;
    }
  }

  getTableFilesNeedingUploaded (
    tableColumns: ReferenceFieldsUI.TableFieldForCrudUi[],
    responseMap: ReferenceFieldsUI.RefResponseMap
  ): ReferenceFieldsUI.FileNeedingUploaded[] {
    const filesNeedingUploaded: ReferenceFieldsUI.FileNeedingUploaded[] = [];
    tableColumns.forEach((column) => {
      const answer = responseMap[column.referenceField.key];
      if (
        answer instanceof Array &&
        column.referenceField.type === ReferenceFieldsUI.ReferenceFieldTypes.FileUpload
      ) {
        answer.forEach((file: any) => {
          if (
            file instanceof YcFile &&
            !file.fileUploadId
          ) {
            filesNeedingUploaded.push({
              key: column.referenceField.key,
              referenceFieldId: column.referenceFieldId,
              file
            });
          }
        });
      }
    });

    return filesNeedingUploaded;
  }

  /**
   * Takes an array of files, uploads them, and returns the updated files
   *
   * @param filesNeedingUploaded array of table files that need to be uploaded
   * @param applicationId application id
   * @param applicationFormId application form id
   * @param column table column row response
   * @returns updated array of files
   */
  async doUploadOfReferenceFieldTableFiles (
    filesNeedingUploaded: ReferenceFieldsUI.FileNeedingUploaded[],
    applicationId: number,
    applicationFormId: number,
    column: ReferenceFieldAPI.ApplicationRefFieldResponse
  ): Promise<YcFile<File>[]> {
    for (const item of filesNeedingUploaded) {
      const newFile = await this.uploadReferenceFieldFile(
        item,
        applicationId,
        applicationFormId
      );
      const columnFiles = column.value as YcFile<File>[];
      const columnValueIndex = columnFiles.findIndex((file) => {
        return file === item.file;
      });

      return [
        ...columnFiles.slice(0, columnValueIndex),
        newFile,
        ...columnFiles.slice(columnValueIndex + 1)
      ];
    }

    return column.value as YcFile<File>[];
  }

  /**
   * Takes an array of files, uploads them, and updates the response map
   *
   * @param filesNeedingUploaded array of files that need to be uploaded
   * @param applicationId application id
   * @param applicationFormId application form id
   * @param refChangeTracker response map that stores the file answer changes
   * @param referenceFieldsMap map that stores all the reference fields answers
   * @returns returns the updated maps
   */
  async doUploadOfReferenceFieldFiles (
    filesNeedingUploaded: ReferenceFieldsUI.FileNeedingUploaded[],
    applicationId: number,
    applicationFormId: number,
    refChangeTracker: ReferenceFieldsUI.RefResponseMap,
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap
  ): Promise<{
    refChangeTracker: ReferenceFieldsUI.RefResponseMap;
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap;
  }> {
    for (const item of filesNeedingUploaded) {
      const file = await this.uploadReferenceFieldFile(
        item,
        applicationId,
        applicationFormId
      );
      refChangeTracker = this.mapNewFileInfoBackToResponseMap(
        item,
        file,
        refChangeTracker
      );
      referenceFieldsMap = this.mapNewFileInfoBackToResponseMap(
        item,
        file,
        referenceFieldsMap
      );
    }

    return {
      refChangeTracker,
      referenceFieldsMap
    };
  }

  /**
   * Actually upload the reference field file
   *
   * @param item details of file to upload
   * @param applicationId application id
   * @param applicationFormId application form id
   * @returns the updated YcFile after upload
   */
  async uploadReferenceFieldFile (
    item: ReferenceFieldsUI.FileNeedingUploaded,
    applicationId: number,
    applicationFormId: number
  ): Promise<YcFile<File>> {
    const file = item.file;
    const fileName = file.fileName;
    const obs = this.applicationFileService.uploadFileWithProgress(
      applicationId,
      applicationFormId,
      file.file,
      fileName,
      item.referenceFieldId
    );
    file.fileUrl = await this.fileUploadProgressService.processFileUploadRequest(
      obs,
      file
    );
    const details = this.applicationFileService.breakDownloadUrlDownToObject(
      file.fileUrl
    );
    if (details.fileId) {
      file.fileUploadId = +details.fileId;
    }

    return file;
  }

  /**
   *
   * @param fileDetails file details for the file that was uploaded
   * @param file new file we uploaded
   * @param referenceFieldsMap response map that needs updated to link to new file
   * @returns the updated reference fields map
   */
  mapNewFileInfoBackToResponseMap (
    fileDetails: ReferenceFieldsUI.FileNeedingUploaded,
    file: YcFile<File>,
    referenceFieldsMap: ReferenceFieldsUI.RefResponseMap
  ) {
    const allFilesForField = referenceFieldsMap[fileDetails.key] as YcFile<File>[] || [];
    const thisFileIndex = allFilesForField.findIndex((_file) => {
      return _file === file;
    });
    const updatedFiles = [
      ...allFilesForField.slice(0, thisFileIndex),
      file,
      ...allFilesForField.slice(thisFileIndex + 1)
    ];
    if (thisFileIndex > -1) {
      referenceFieldsMap = {
        ...referenceFieldsMap,
        [fileDetails.key]: updatedFiles
      };
    }

    return referenceFieldsMap;
  }

  async handleFieldTypeConversion (
    referenceFieldId: number,
    conversionType: ReferenceFieldsUI.RefFieldConversionTypes
  ) {
    try {
      switch (conversionType) {
        case ReferenceFieldsUI.RefFieldConversionTypes.NUMBER_TO_TEXT:
          await this.referenceFieldsResources.convertNumberFieldToText(referenceFieldId);
          break;
        case ReferenceFieldsUI.RefFieldConversionTypes.TEXT_TO_NUMBER:
          await this.referenceFieldsResources.convertTextFieldToNumber(referenceFieldId);
          break;
        case ReferenceFieldsUI.RefFieldConversionTypes.DATE_TO_TEXT:
          await this.referenceFieldsResources.convertDateFieldToText(referenceFieldId);
          break;
        case ReferenceFieldsUI.RefFieldConversionTypes.TEXT_TO_DATE:
          await this.referenceFieldsResources.convertTextFieldToDate(referenceFieldId);
          break;
      }
      await this.resetFieldsAndCategories();
      this.notifier.success(this.i18n.translate(
        'FORMS:textSuccessfullyConvertedFormField',
        {},
        'Successfully converted form field')
      );
    } catch (e) {
      this.logger.error(e);
      this.notifier.error(
        this.i18n.translate(
          'FORMS:textThereWasAnErrorConvertingTheFieldType',
          {},
          'There was an error converting the field type'
        )
      );
    }
  }

  async validateFieldTypeConversion (
    referenceFieldId: number,
    conversionType: ReferenceFieldsUI.RefFieldConversionTypes
  ) {
    let response;
    switch (conversionType) {
      case ReferenceFieldsUI.RefFieldConversionTypes.NUMBER_TO_TEXT:
        response = await this.referenceFieldsResources.validateNumberToTextFieldConversion(referenceFieldId);
        break;
      case ReferenceFieldsUI.RefFieldConversionTypes.TEXT_TO_NUMBER:
        response = await this.referenceFieldsResources.validateTextToNumberFieldConversion(referenceFieldId);
        break;
      case ReferenceFieldsUI.RefFieldConversionTypes.DATE_TO_TEXT:
        response = await this.referenceFieldsResources.validateDateToTextFieldConversion(referenceFieldId);
        break;
      case ReferenceFieldsUI.RefFieldConversionTypes.TEXT_TO_DATE:
        response = await this.referenceFieldsResources.validateTextToDateFieldConversion(referenceFieldId);
        break;
    }

    return response;
  }

  /**
   * Returns the number of columns on the subset or table field
   *
   * @param field: the reference field
   * @returns the number of columsn (for tables and subsets)
   */
  getNumberOfColumns (field: ReferenceFieldAPI.ReferenceFieldDisplayModel) {
    if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Subset) {
      return this.dataPointsMap[field.referenceFieldId]?.length ?? 0;
    } else if (field?.type === ReferenceFieldsUI.ReferenceFieldTypes.Table) {
      return this.tableColumnsMap[field.referenceFieldId]?.length ?? 0;
    }

    return 0;
  }
}