Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | 5x 5x 5x 5x 5x 5x 129x 5x 5x 5x 38x 1x 37x 37x 37x 5x 93x 93x 5x 76x 2x 74x 74x 19x 5x 5x 8x 4x 4x 4x 4x 4x 4x 4x 3x 8x 8x 8x 8x 8x 8x 7x 1x 8x 48x 48x 48x 55x 55x 49x 49x 49x 45x 45x 4x 5x 8x 8x 8x 8x 43x 43x 8x 5x 5x 13x 13x 12x 12x 12x 12x 12x 12x 12x 12x 12x 12x 24x 24x 2x 22x 13x 3x 3x 13x 13x 22x 12x 10x 12x 12x 13x 5x | import { set, uniq } from "lodash";
import { hashObject } from "./common";
import { getDB } from "./db";
export const BLOCK_SIZE = 60e3 * 60 * 24; // 1 day
// d = new Date('+050705-08-09T23:40:06.178Z')
// d.getTime() === 1537947128406178
// Math.floor(1537947128406178 / BLOCK_SIZE) === 17800313
// max block number: 17800313 =>
// fixed block id length is 8 digits preceded by a 'B' => 9 chars
const BLOCK_ID_LENGTH = 8;
export function getBlockId(modified: number) {
return 'B' + String(Math.floor(modified / BLOCK_SIZE)).padStart(BLOCK_ID_LENGTH, "0");
}
const MIN_BLOCK_ID = getBlockId(0);
const MAX_BLOCK_ID = getBlockId(1537947128406178);
export function getBlockRange(blockId: string) {
if (blockId === 'users') {
return {
min: -Infinity,
max: Infinity,
}
}
const blockNum = Number(blockId.substring(1));
const min = blockNum * BLOCK_SIZE;
return {
min,
max: min + BLOCK_SIZE,
}
}
export const invalidatedBlockIds: { [groupId: string]: { [blockId: string]: true } } = {};
function invalidateCacheForModified(groupId: string, modified: number) {
let blockId = getBlockId(modified);
// invalidatedBlockIds[groupId][blockId] = true;
set(invalidatedBlockIds, `${groupId}.${blockId}`, true);
}
export function invalidateCache(groupId: string, modified: number, oldModified?: number) {
if (groupId === 'users') {
// users are synced in a different way so we don't need to worry about maintaining the cache for them
// NOTE: I'm not sure this is true, we do sync users for the group but I think we're sending partial changes for users so the users' aren't signed which is no good
return;
}
invalidateCacheForModified(groupId, modified);
if (oldModified) {
invalidateCacheForModified(groupId, oldModified);
}
}
export const prefixHashDetails: {
[groupId: string]: {
[prefix: string]: {
[subPrefix: string]: string
}
}
} = {};
export async function populateGroupHashes(groupId: string) {
let blockIds: string[];
let blockIdHashes: { [blockId: string]: string };
if (!prefixHashDetails[groupId]) {
// initial populate
blockIdHashes = await getDetailHashes(groupId);
blockIds = Object.keys(blockIdHashes);
invalidatedBlockIds[groupId] = {};
} else {
// recompute any invalidated blockIds
// TODO theoretically there could be a situation where we don't want to repopulate _all_ invalidated blockIds
blockIds = Object.keys(invalidatedBlockIds[groupId]);
blockIdHashes = {};
invalidatedBlockIds[groupId] = {};
for (const blockId of blockIds) {
blockIdHashes[blockId] = (await getDetailHashes(groupId, blockId))[blockId];
}
}
let parents: string[] = [];
for (const blockId of blockIds) {
const hash = blockIdHashes[blockId];
const parent = blockId.substring(0, blockId.length - 1);
parents.push(parent);
if (hash) {
set(prefixHashDetails, `${groupId}.${parent}.${blockId}`, hash);
} else {
delete prefixHashDetails?.[groupId]?.[parent]?.[blockId];
}
}
while (parents.length) {
const prefixes = uniq(parents);
parents.length = 0;
for (const prefix of prefixes) {
const parent = prefix.substring(0, prefix.length - 1);
if (!parent) break;
parents.push(parent);
const hashDetails = prefixHashDetails[groupId][prefix];
if (Object.keys(hashDetails).length > 0) {
const hash = hashObject(hashDetails);
set(prefixHashDetails, `${groupId}.${parent}.${prefix}`, hash);
} else {
delete prefixHashDetails[groupId]?.[parent]?.[prefix];
}
}
}
// console.log(prefixHashDetails[groupId])
}
export async function getPrefixHashes(groupId: string, blockPrefix = 'B'): Promise<{ [subPrefix: string]: string }> {
await populateGroupHashes(groupId);
let hashes = prefixHashDetails[groupId]?.[blockPrefix] ?? {};
// if (!hashes) ...
let prefixes: string[] = Object.keys(hashes);
while (prefixes.length === 1 && prefixes[0].length < 9) {
hashes = prefixHashDetails[groupId][prefixes[0]];
prefixes = Object.keys(hashes);
}
return hashes;
}
let getDetailHashPromises: { [promiseId: string]: Promise<{ [blockId: string]: string; }> } = {};
export function getDetailHashes(groupId: string, blockId_?: string) {
const promiseId = `${groupId}-${blockId_ || "all"}`;
if (!getDetailHashPromises[promiseId]) {
getDetailHashPromises[promiseId] = new Promise(async resolve => {
const minModified = getBlockRange(blockId_ || MIN_BLOCK_ID).min;
const maxModified = getBlockRange(blockId_ || MAX_BLOCK_ID).max;
const db = await getDB();
const cursor = await db.changes.openCursor(groupId, minModified);
const data: { id: string, modified: number }[] = [];
let blockId: string = "";
let blockUpperModified = -Infinity;
const hashes: { [blockId: string]: string } = {};
while (await cursor.next()) {
const change = cursor.value;
if (change.modified > maxModified) {
break;
}
if (blockUpperModified < change.modified) {
if (data.length) {
hashes[blockId] = hashObject(data);
data.length = 0;
}
blockId = getBlockId(change.modified);
blockUpperModified = getBlockRange(blockId).max;
}
data.push({ id: change.id, modified: change.modified });
}
if (data.length) {
hashes[blockId] = hashObject(data);
}
resolve(hashes);
delete getDetailHashPromises[promiseId];
});
}
return getDetailHashPromises[promiseId];
}
export async function getBlockChangeInfo(groupId: string, blockId: string) {
const range = getBlockRange(blockId);
const minModified = range.min;
const maxModified = range.max;
const db = await getDB();
const cursor = await db.changes.openCursor(groupId, minModified);
const data: { id: string, modified: number}[] = [];
while (await cursor.next()) {
const change = cursor.value;
Iif (change.modified > maxModified) {
break;
}
data.push({ id: cursor.value.id, modified: cursor.value.modified });
}
return data;
} |