BinaryMiddleware.js 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. */
  4. "use strict";
  5. const memoize = require("../util/memoize");
  6. const SerializerMiddleware = require("./SerializerMiddleware");
  7. /** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
  8. /** @typedef {import("./types").PrimitiveSerializableType} PrimitiveSerializableType */
  9. /*
  10. Format:
  11. File -> Section*
  12. Section -> NullsSection |
  13. BooleansSection |
  14. F64NumbersSection |
  15. I32NumbersSection |
  16. I8NumbersSection |
  17. ShortStringSection |
  18. BigIntSection |
  19. I32BigIntSection |
  20. I8BigIntSection
  21. StringSection |
  22. BufferSection |
  23. NopSection
  24. NullsSection ->
  25. NullHeaderByte | Null2HeaderByte | Null3HeaderByte |
  26. Nulls8HeaderByte 0xnn (n:count - 4) |
  27. Nulls32HeaderByte n:ui32 (n:count - 260) |
  28. BooleansSection -> TrueHeaderByte | FalseHeaderByte | BooleansSectionHeaderByte BooleansCountAndBitsByte
  29. F64NumbersSection -> F64NumbersSectionHeaderByte f64*
  30. I32NumbersSection -> I32NumbersSectionHeaderByte i32*
  31. I8NumbersSection -> I8NumbersSectionHeaderByte i8*
  32. ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
  33. StringSection -> StringSectionHeaderByte i32:length utf8-byte*
  34. BufferSection -> BufferSectionHeaderByte i32:length byte*
  35. NopSection --> NopSectionHeaderByte
  36. BigIntSection -> BigIntSectionHeaderByte i32:length ascii-byte*
  37. I32BigIntSection -> I32BigIntSectionHeaderByte i32
  38. I8BigIntSection -> I8BigIntSectionHeaderByte i8
  39. ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)
  40. F64NumbersSectionHeaderByte -> 0b001n_nnnn (n:count - 1)
  41. I32NumbersSectionHeaderByte -> 0b010n_nnnn (n:count - 1)
  42. I8NumbersSectionHeaderByte -> 0b011n_nnnn (n:count - 1)
  43. NullsSectionHeaderByte -> 0b0001_nnnn (n:count - 1)
  44. BooleansCountAndBitsByte ->
  45. 0b0000_1xxx (count = 3) |
  46. 0b0001_xxxx (count = 4) |
  47. 0b001x_xxxx (count = 5) |
  48. 0b01xx_xxxx (count = 6) |
  49. 0b1nnn_nnnn (n:count - 7, 7 <= count <= 133)
  50. 0xff n:ui32 (n:count, 134 <= count < 2^32)
  51. StringSectionHeaderByte -> 0b0000_1110
  52. BufferSectionHeaderByte -> 0b0000_1111
  53. NopSectionHeaderByte -> 0b0000_1011
  54. BigIntSectionHeaderByte -> 0b0001_1010
  55. I32BigIntSectionHeaderByte -> 0b0001_1100
  56. I8BigIntSectionHeaderByte -> 0b0001_1011
  57. FalseHeaderByte -> 0b0000_1100
  58. TrueHeaderByte -> 0b0000_1101
  59. RawNumber -> n (n <= 10)
  60. */
  61. const LAZY_HEADER = 0x0b;
  62. const TRUE_HEADER = 0x0c;
  63. const FALSE_HEADER = 0x0d;
  64. const BOOLEANS_HEADER = 0x0e;
  65. const NULL_HEADER = 0x10;
  66. const NULL2_HEADER = 0x11;
  67. const NULL3_HEADER = 0x12;
  68. const NULLS8_HEADER = 0x13;
  69. const NULLS32_HEADER = 0x14;
  70. const NULL_AND_I8_HEADER = 0x15;
  71. const NULL_AND_I32_HEADER = 0x16;
  72. const NULL_AND_TRUE_HEADER = 0x17;
  73. const NULL_AND_FALSE_HEADER = 0x18;
  74. const BIGINT_HEADER = 0x1a;
  75. const BIGINT_I8_HEADER = 0x1b;
  76. const BIGINT_I32_HEADER = 0x1c;
  77. const STRING_HEADER = 0x1e;
  78. const BUFFER_HEADER = 0x1f;
  79. const I8_HEADER = 0x60;
  80. const I32_HEADER = 0x40;
  81. const F64_HEADER = 0x20;
  82. const SHORT_STRING_HEADER = 0x80;
  83. /** Uplift high-order bits */
  84. const NUMBERS_HEADER_MASK = 0xe0; // 0b1010_0000
  85. const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
  86. const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111
  87. const HEADER_SIZE = 1;
  88. const I8_SIZE = 1;
  89. const I32_SIZE = 4;
  90. const F64_SIZE = 8;
  91. const MEASURE_START_OPERATION = Symbol("MEASURE_START_OPERATION");
  92. const MEASURE_END_OPERATION = Symbol("MEASURE_END_OPERATION");
  93. /** @typedef {typeof MEASURE_START_OPERATION} MEASURE_START_OPERATION_TYPE */
  94. /** @typedef {typeof MEASURE_END_OPERATION} MEASURE_END_OPERATION_TYPE */
  95. /**
  96. * @param {number} n number
  97. * @returns {0 | 1 | 2} type of number for serialization
  98. */
  99. const identifyNumber = n => {
  100. if (n === (n | 0)) {
  101. if (n <= 127 && n >= -128) return 0;
  102. if (n <= 2147483647 && n >= -2147483648) return 1;
  103. }
  104. return 2;
  105. };
  106. /**
  107. * @param {bigint} n bigint
  108. * @returns {0 | 1 | 2} type of bigint for serialization
  109. */
  110. const identifyBigInt = n => {
  111. if (n <= BigInt(127) && n >= BigInt(-128)) return 0;
  112. if (n <= BigInt(2147483647) && n >= BigInt(-2147483648)) return 1;
  113. return 2;
  114. };
  115. /** @typedef {TODO} Context */
  116. /**
  117. * @typedef {PrimitiveSerializableType[]} DeserializedType
  118. * @typedef {BufferSerializableType[]} SerializedType
  119. * @extends {SerializerMiddleware<DeserializedType, SerializedType>}
  120. */
  121. class BinaryMiddleware extends SerializerMiddleware {
  122. /**
  123. * @param {DeserializedType} data data
  124. * @param {object} context context object
  125. * @returns {SerializedType|Promise<SerializedType>} serialized data
  126. */
  127. serialize(data, context) {
  128. return this._serialize(data, context);
  129. }
  130. /**
  131. * @param {function(): Promise<any> | any} fn lazy function
  132. * @param {TODO} context serialize function
  133. * @returns {function(): Promise<any> | any} new lazy
  134. */
  135. _serializeLazy(fn, context) {
  136. return SerializerMiddleware.serializeLazy(fn, data =>
  137. this._serialize(data, context)
  138. );
  139. }
  140. /**
  141. * @param {DeserializedType} data data
  142. * @param {TODO} context context object
  143. * @param {{ leftOverBuffer: Buffer | null, allocationSize: number, increaseCounter: number }} allocationScope allocation scope
  144. * @returns {SerializedType} serialized data
  145. */
  146. _serialize(
  147. data,
  148. context,
  149. allocationScope = {
  150. allocationSize: 1024,
  151. increaseCounter: 0,
  152. leftOverBuffer: null
  153. }
  154. ) {
  155. /** @type {Buffer | null} */
  156. let leftOverBuffer = null;
  157. /** @type {BufferSerializableType[]} */
  158. let buffers = [];
  159. /** @type {Buffer | null} */
  160. let currentBuffer = allocationScope ? allocationScope.leftOverBuffer : null;
  161. allocationScope.leftOverBuffer = null;
  162. let currentPosition = 0;
  163. if (currentBuffer === null) {
  164. currentBuffer = Buffer.allocUnsafe(allocationScope.allocationSize);
  165. }
  166. /**
  167. * @param {number} bytesNeeded bytes needed
  168. */
  169. const allocate = bytesNeeded => {
  170. if (currentBuffer !== null) {
  171. if (currentBuffer.length - currentPosition >= bytesNeeded) return;
  172. flush();
  173. }
  174. if (leftOverBuffer && leftOverBuffer.length >= bytesNeeded) {
  175. currentBuffer = leftOverBuffer;
  176. leftOverBuffer = null;
  177. } else {
  178. currentBuffer = Buffer.allocUnsafe(
  179. Math.max(bytesNeeded, allocationScope.allocationSize)
  180. );
  181. if (
  182. !(allocationScope.increaseCounter =
  183. (allocationScope.increaseCounter + 1) % 4) &&
  184. allocationScope.allocationSize < 16777216
  185. ) {
  186. allocationScope.allocationSize = allocationScope.allocationSize << 1;
  187. }
  188. }
  189. };
  190. const flush = () => {
  191. if (currentBuffer !== null) {
  192. if (currentPosition > 0) {
  193. buffers.push(
  194. Buffer.from(
  195. currentBuffer.buffer,
  196. currentBuffer.byteOffset,
  197. currentPosition
  198. )
  199. );
  200. }
  201. if (
  202. !leftOverBuffer ||
  203. leftOverBuffer.length < currentBuffer.length - currentPosition
  204. ) {
  205. leftOverBuffer = Buffer.from(
  206. currentBuffer.buffer,
  207. currentBuffer.byteOffset + currentPosition,
  208. currentBuffer.byteLength - currentPosition
  209. );
  210. }
  211. currentBuffer = null;
  212. currentPosition = 0;
  213. }
  214. };
  215. /**
  216. * @param {number} byte byte
  217. */
  218. const writeU8 = byte => {
  219. /** @type {Buffer} */
  220. (currentBuffer).writeUInt8(byte, currentPosition++);
  221. };
  222. /**
  223. * @param {number} ui32 ui32
  224. */
  225. const writeU32 = ui32 => {
  226. /** @type {Buffer} */
  227. (currentBuffer).writeUInt32LE(ui32, currentPosition);
  228. currentPosition += 4;
  229. };
  230. /** @type {number[]} */
  231. const measureStack = [];
  232. const measureStart = () => {
  233. measureStack.push(buffers.length, currentPosition);
  234. };
  235. /**
  236. * @returns {number} size
  237. */
  238. const measureEnd = () => {
  239. const oldPos = /** @type {number} */ (measureStack.pop());
  240. const buffersIndex = /** @type {number} */ (measureStack.pop());
  241. let size = currentPosition - oldPos;
  242. for (let i = buffersIndex; i < buffers.length; i++) {
  243. size += buffers[i].length;
  244. }
  245. return size;
  246. };
  247. for (let i = 0; i < data.length; i++) {
  248. const thing = data[i];
  249. switch (typeof thing) {
  250. case "function": {
  251. if (!SerializerMiddleware.isLazy(thing))
  252. throw new Error(`Unexpected function ${thing}`);
  253. /** @type {SerializedType | (() => SerializedType)} */
  254. let serializedData =
  255. SerializerMiddleware.getLazySerializedValue(thing);
  256. if (serializedData === undefined) {
  257. if (SerializerMiddleware.isLazy(thing, this)) {
  258. flush();
  259. allocationScope.leftOverBuffer = leftOverBuffer;
  260. const result =
  261. /** @type {(Exclude<PrimitiveSerializableType, Promise<PrimitiveSerializableType>>)[]} */ (
  262. thing()
  263. );
  264. const data = this._serialize(result, context, allocationScope);
  265. leftOverBuffer = allocationScope.leftOverBuffer;
  266. allocationScope.leftOverBuffer = null;
  267. SerializerMiddleware.setLazySerializedValue(thing, data);
  268. serializedData = data;
  269. } else {
  270. serializedData = this._serializeLazy(thing, context);
  271. flush();
  272. buffers.push(serializedData);
  273. break;
  274. }
  275. } else if (typeof serializedData === "function") {
  276. flush();
  277. buffers.push(serializedData);
  278. break;
  279. }
  280. /** @type {number[]} */
  281. const lengths = [];
  282. for (const item of serializedData) {
  283. let last;
  284. if (typeof item === "function") {
  285. lengths.push(0);
  286. } else if (item.length === 0) {
  287. // ignore
  288. } else if (
  289. lengths.length > 0 &&
  290. (last = lengths[lengths.length - 1]) !== 0
  291. ) {
  292. const remaining = 0xffffffff - last;
  293. if (remaining >= item.length) {
  294. lengths[lengths.length - 1] += item.length;
  295. } else {
  296. lengths.push(item.length - remaining);
  297. lengths[lengths.length - 2] = 0xffffffff;
  298. }
  299. } else {
  300. lengths.push(item.length);
  301. }
  302. }
  303. allocate(5 + lengths.length * 4);
  304. writeU8(LAZY_HEADER);
  305. writeU32(lengths.length);
  306. for (const l of lengths) {
  307. writeU32(l);
  308. }
  309. flush();
  310. for (const item of serializedData) {
  311. buffers.push(item);
  312. }
  313. break;
  314. }
  315. case "string": {
  316. const len = Buffer.byteLength(thing);
  317. if (len >= 128 || len !== thing.length) {
  318. allocate(len + HEADER_SIZE + I32_SIZE);
  319. writeU8(STRING_HEADER);
  320. writeU32(len);
  321. currentBuffer.write(thing, currentPosition);
  322. currentPosition += len;
  323. } else if (len >= 70) {
  324. allocate(len + HEADER_SIZE);
  325. writeU8(SHORT_STRING_HEADER | len);
  326. currentBuffer.write(thing, currentPosition, "latin1");
  327. currentPosition += len;
  328. } else {
  329. allocate(len + HEADER_SIZE);
  330. writeU8(SHORT_STRING_HEADER | len);
  331. for (let i = 0; i < len; i++) {
  332. currentBuffer[currentPosition++] = thing.charCodeAt(i);
  333. }
  334. }
  335. break;
  336. }
  337. case "bigint": {
  338. const type = identifyBigInt(thing);
  339. if (type === 0 && thing >= 0 && thing <= BigInt(10)) {
  340. // shortcut for very small bigints
  341. allocate(HEADER_SIZE + I8_SIZE);
  342. writeU8(BIGINT_I8_HEADER);
  343. writeU8(Number(thing));
  344. break;
  345. }
  346. switch (type) {
  347. case 0: {
  348. let n = 1;
  349. allocate(HEADER_SIZE + I8_SIZE * n);
  350. writeU8(BIGINT_I8_HEADER | (n - 1));
  351. while (n > 0) {
  352. currentBuffer.writeInt8(
  353. Number(/** @type {bigint} */ (data[i])),
  354. currentPosition
  355. );
  356. currentPosition += I8_SIZE;
  357. n--;
  358. i++;
  359. }
  360. i--;
  361. break;
  362. }
  363. case 1: {
  364. let n = 1;
  365. allocate(HEADER_SIZE + I32_SIZE * n);
  366. writeU8(BIGINT_I32_HEADER | (n - 1));
  367. while (n > 0) {
  368. currentBuffer.writeInt32LE(
  369. Number(/** @type {bigint} */ (data[i])),
  370. currentPosition
  371. );
  372. currentPosition += I32_SIZE;
  373. n--;
  374. i++;
  375. }
  376. i--;
  377. break;
  378. }
  379. default: {
  380. const value = thing.toString();
  381. const len = Buffer.byteLength(value);
  382. allocate(len + HEADER_SIZE + I32_SIZE);
  383. writeU8(BIGINT_HEADER);
  384. writeU32(len);
  385. currentBuffer.write(value, currentPosition);
  386. currentPosition += len;
  387. break;
  388. }
  389. }
  390. break;
  391. }
  392. case "number": {
  393. const type = identifyNumber(thing);
  394. if (type === 0 && thing >= 0 && thing <= 10) {
  395. // shortcut for very small numbers
  396. allocate(I8_SIZE);
  397. writeU8(thing);
  398. break;
  399. }
  400. /**
  401. * amount of numbers to write
  402. * @type {number}
  403. */
  404. let n = 1;
  405. for (; n < 32 && i + n < data.length; n++) {
  406. const item = data[i + n];
  407. if (typeof item !== "number") break;
  408. if (identifyNumber(item) !== type) break;
  409. }
  410. switch (type) {
  411. case 0:
  412. allocate(HEADER_SIZE + I8_SIZE * n);
  413. writeU8(I8_HEADER | (n - 1));
  414. while (n > 0) {
  415. currentBuffer.writeInt8(
  416. /** @type {number} */ (data[i]),
  417. currentPosition
  418. );
  419. currentPosition += I8_SIZE;
  420. n--;
  421. i++;
  422. }
  423. break;
  424. case 1:
  425. allocate(HEADER_SIZE + I32_SIZE * n);
  426. writeU8(I32_HEADER | (n - 1));
  427. while (n > 0) {
  428. currentBuffer.writeInt32LE(
  429. /** @type {number} */ (data[i]),
  430. currentPosition
  431. );
  432. currentPosition += I32_SIZE;
  433. n--;
  434. i++;
  435. }
  436. break;
  437. case 2:
  438. allocate(HEADER_SIZE + F64_SIZE * n);
  439. writeU8(F64_HEADER | (n - 1));
  440. while (n > 0) {
  441. currentBuffer.writeDoubleLE(
  442. /** @type {number} */ (data[i]),
  443. currentPosition
  444. );
  445. currentPosition += F64_SIZE;
  446. n--;
  447. i++;
  448. }
  449. break;
  450. }
  451. i--;
  452. break;
  453. }
  454. case "boolean": {
  455. let lastByte = thing === true ? 1 : 0;
  456. const bytes = [];
  457. let count = 1;
  458. let n;
  459. for (n = 1; n < 0xffffffff && i + n < data.length; n++) {
  460. const item = data[i + n];
  461. if (typeof item !== "boolean") break;
  462. const pos = count & 0x7;
  463. if (pos === 0) {
  464. bytes.push(lastByte);
  465. lastByte = item === true ? 1 : 0;
  466. } else if (item === true) {
  467. lastByte |= 1 << pos;
  468. }
  469. count++;
  470. }
  471. i += count - 1;
  472. if (count === 1) {
  473. allocate(HEADER_SIZE);
  474. writeU8(lastByte === 1 ? TRUE_HEADER : FALSE_HEADER);
  475. } else if (count === 2) {
  476. allocate(HEADER_SIZE * 2);
  477. writeU8(lastByte & 1 ? TRUE_HEADER : FALSE_HEADER);
  478. writeU8(lastByte & 2 ? TRUE_HEADER : FALSE_HEADER);
  479. } else if (count <= 6) {
  480. allocate(HEADER_SIZE + I8_SIZE);
  481. writeU8(BOOLEANS_HEADER);
  482. writeU8((1 << count) | lastByte);
  483. } else if (count <= 133) {
  484. allocate(HEADER_SIZE + I8_SIZE + I8_SIZE * bytes.length + I8_SIZE);
  485. writeU8(BOOLEANS_HEADER);
  486. writeU8(0x80 | (count - 7));
  487. for (const byte of bytes) writeU8(byte);
  488. writeU8(lastByte);
  489. } else {
  490. allocate(
  491. HEADER_SIZE +
  492. I8_SIZE +
  493. I32_SIZE +
  494. I8_SIZE * bytes.length +
  495. I8_SIZE
  496. );
  497. writeU8(BOOLEANS_HEADER);
  498. writeU8(0xff);
  499. writeU32(count);
  500. for (const byte of bytes) writeU8(byte);
  501. writeU8(lastByte);
  502. }
  503. break;
  504. }
  505. case "object": {
  506. if (thing === null) {
  507. let n;
  508. for (n = 1; n < 0x100000104 && i + n < data.length; n++) {
  509. const item = data[i + n];
  510. if (item !== null) break;
  511. }
  512. i += n - 1;
  513. if (n === 1) {
  514. if (i + 1 < data.length) {
  515. const next = data[i + 1];
  516. if (next === true) {
  517. allocate(HEADER_SIZE);
  518. writeU8(NULL_AND_TRUE_HEADER);
  519. i++;
  520. } else if (next === false) {
  521. allocate(HEADER_SIZE);
  522. writeU8(NULL_AND_FALSE_HEADER);
  523. i++;
  524. } else if (typeof next === "number") {
  525. const type = identifyNumber(next);
  526. if (type === 0) {
  527. allocate(HEADER_SIZE + I8_SIZE);
  528. writeU8(NULL_AND_I8_HEADER);
  529. currentBuffer.writeInt8(next, currentPosition);
  530. currentPosition += I8_SIZE;
  531. i++;
  532. } else if (type === 1) {
  533. allocate(HEADER_SIZE + I32_SIZE);
  534. writeU8(NULL_AND_I32_HEADER);
  535. currentBuffer.writeInt32LE(next, currentPosition);
  536. currentPosition += I32_SIZE;
  537. i++;
  538. } else {
  539. allocate(HEADER_SIZE);
  540. writeU8(NULL_HEADER);
  541. }
  542. } else {
  543. allocate(HEADER_SIZE);
  544. writeU8(NULL_HEADER);
  545. }
  546. } else {
  547. allocate(HEADER_SIZE);
  548. writeU8(NULL_HEADER);
  549. }
  550. } else if (n === 2) {
  551. allocate(HEADER_SIZE);
  552. writeU8(NULL2_HEADER);
  553. } else if (n === 3) {
  554. allocate(HEADER_SIZE);
  555. writeU8(NULL3_HEADER);
  556. } else if (n < 260) {
  557. allocate(HEADER_SIZE + I8_SIZE);
  558. writeU8(NULLS8_HEADER);
  559. writeU8(n - 4);
  560. } else {
  561. allocate(HEADER_SIZE + I32_SIZE);
  562. writeU8(NULLS32_HEADER);
  563. writeU32(n - 260);
  564. }
  565. } else if (Buffer.isBuffer(thing)) {
  566. if (thing.length < 8192) {
  567. allocate(HEADER_SIZE + I32_SIZE + thing.length);
  568. writeU8(BUFFER_HEADER);
  569. writeU32(thing.length);
  570. thing.copy(currentBuffer, currentPosition);
  571. currentPosition += thing.length;
  572. } else {
  573. allocate(HEADER_SIZE + I32_SIZE);
  574. writeU8(BUFFER_HEADER);
  575. writeU32(thing.length);
  576. flush();
  577. buffers.push(thing);
  578. }
  579. }
  580. break;
  581. }
  582. case "symbol": {
  583. if (thing === MEASURE_START_OPERATION) {
  584. measureStart();
  585. } else if (thing === MEASURE_END_OPERATION) {
  586. const size = measureEnd();
  587. allocate(HEADER_SIZE + I32_SIZE);
  588. writeU8(I32_HEADER);
  589. currentBuffer.writeInt32LE(size, currentPosition);
  590. currentPosition += I32_SIZE;
  591. }
  592. break;
  593. }
  594. default: {
  595. throw new Error(
  596. `Unknown typeof "${typeof thing}" in binary middleware`
  597. );
  598. }
  599. }
  600. }
  601. flush();
  602. allocationScope.leftOverBuffer = leftOverBuffer;
  603. // avoid leaking memory
  604. currentBuffer = null;
  605. leftOverBuffer = null;
  606. allocationScope = /** @type {EXPECTED_ANY} */ (undefined);
  607. const _buffers = buffers;
  608. buffers = /** @type {EXPECTED_ANY} */ (undefined);
  609. return _buffers;
  610. }
  611. /**
  612. * @param {SerializedType} data data
  613. * @param {object} context context object
  614. * @returns {DeserializedType|Promise<DeserializedType>} deserialized data
  615. */
  616. deserialize(data, context) {
  617. return this._deserialize(data, context);
  618. }
  619. _createLazyDeserialized(content, context) {
  620. return SerializerMiddleware.createLazy(
  621. memoize(() => this._deserialize(content, context)),
  622. this,
  623. undefined,
  624. content
  625. );
  626. }
  627. _deserializeLazy(fn, context) {
  628. return SerializerMiddleware.deserializeLazy(fn, data =>
  629. this._deserialize(data, context)
  630. );
  631. }
  632. /**
  633. * @param {SerializedType} data data
  634. * @param {TODO} context context object
  635. * @returns {DeserializedType} deserialized data
  636. */
  637. _deserialize(data, context) {
  638. let currentDataItem = 0;
  639. /** @type {BufferSerializableType | null} */
  640. let currentBuffer = data[0];
  641. let currentIsBuffer = Buffer.isBuffer(currentBuffer);
  642. let currentPosition = 0;
  643. /** @type {(x: Buffer) => Buffer} */
  644. const retainedBuffer = context.retainedBuffer || (x => x);
  645. const checkOverflow = () => {
  646. if (currentPosition >= /** @type {Buffer} */ (currentBuffer).length) {
  647. currentPosition = 0;
  648. currentDataItem++;
  649. currentBuffer =
  650. currentDataItem < data.length ? data[currentDataItem] : null;
  651. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  652. }
  653. };
  654. /**
  655. * @param {number} n n
  656. * @returns {boolean} true when in current buffer, otherwise false
  657. */
  658. const isInCurrentBuffer = n =>
  659. currentIsBuffer &&
  660. n + currentPosition <= /** @type {Buffer} */ (currentBuffer).length;
  661. const ensureBuffer = () => {
  662. if (!currentIsBuffer) {
  663. throw new Error(
  664. currentBuffer === null
  665. ? "Unexpected end of stream"
  666. : "Unexpected lazy element in stream"
  667. );
  668. }
  669. };
  670. /**
  671. * Reads n bytes
  672. * @param {number} n amount of bytes to read
  673. * @returns {Buffer} buffer with bytes
  674. */
  675. const read = n => {
  676. ensureBuffer();
  677. const rem =
  678. /** @type {Buffer} */ (currentBuffer).length - currentPosition;
  679. if (rem < n) {
  680. const buffers = [read(rem)];
  681. n -= rem;
  682. ensureBuffer();
  683. while (/** @type {Buffer} */ (currentBuffer).length < n) {
  684. const b = /** @type {Buffer} */ (currentBuffer);
  685. buffers.push(b);
  686. n -= b.length;
  687. currentDataItem++;
  688. currentBuffer =
  689. currentDataItem < data.length ? data[currentDataItem] : null;
  690. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  691. ensureBuffer();
  692. }
  693. buffers.push(read(n));
  694. return Buffer.concat(buffers);
  695. }
  696. const b = /** @type {Buffer} */ (currentBuffer);
  697. const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
  698. currentPosition += n;
  699. checkOverflow();
  700. return res;
  701. };
  702. /**
  703. * Reads up to n bytes
  704. * @param {number} n amount of bytes to read
  705. * @returns {Buffer} buffer with bytes
  706. */
  707. const readUpTo = n => {
  708. ensureBuffer();
  709. const rem =
  710. /** @type {Buffer} */
  711. (currentBuffer).length - currentPosition;
  712. if (rem < n) {
  713. n = rem;
  714. }
  715. const b = /** @type {Buffer} */ (currentBuffer);
  716. const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
  717. currentPosition += n;
  718. checkOverflow();
  719. return res;
  720. };
  721. /**
  722. * @returns {number} U8
  723. */
  724. const readU8 = () => {
  725. ensureBuffer();
  726. /**
  727. * There is no need to check remaining buffer size here
  728. * since {@link checkOverflow} guarantees at least one byte remaining
  729. */
  730. const byte =
  731. /** @type {Buffer} */
  732. (currentBuffer).readUInt8(currentPosition);
  733. currentPosition += I8_SIZE;
  734. checkOverflow();
  735. return byte;
  736. };
  737. /**
  738. * @returns {number} U32
  739. */
  740. const readU32 = () => read(I32_SIZE).readUInt32LE(0);
  741. /**
  742. * @param {number} data data
  743. * @param {number} n n
  744. */
  745. const readBits = (data, n) => {
  746. let mask = 1;
  747. while (n !== 0) {
  748. result.push((data & mask) !== 0);
  749. mask = mask << 1;
  750. n--;
  751. }
  752. };
  753. const dispatchTable = Array.from({ length: 256 }).map((_, header) => {
  754. switch (header) {
  755. case LAZY_HEADER:
  756. return () => {
  757. const count = readU32();
  758. const lengths = Array.from({ length: count }).map(() => readU32());
  759. const content = [];
  760. for (let l of lengths) {
  761. if (l === 0) {
  762. if (typeof currentBuffer !== "function") {
  763. throw new Error("Unexpected non-lazy element in stream");
  764. }
  765. content.push(currentBuffer);
  766. currentDataItem++;
  767. currentBuffer =
  768. currentDataItem < data.length ? data[currentDataItem] : null;
  769. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  770. } else {
  771. do {
  772. const buf = readUpTo(l);
  773. l -= buf.length;
  774. content.push(retainedBuffer(buf));
  775. } while (l > 0);
  776. }
  777. }
  778. result.push(this._createLazyDeserialized(content, context));
  779. };
  780. case BUFFER_HEADER:
  781. return () => {
  782. const len = readU32();
  783. result.push(retainedBuffer(read(len)));
  784. };
  785. case TRUE_HEADER:
  786. return () => result.push(true);
  787. case FALSE_HEADER:
  788. return () => result.push(false);
  789. case NULL3_HEADER:
  790. return () => result.push(null, null, null);
  791. case NULL2_HEADER:
  792. return () => result.push(null, null);
  793. case NULL_HEADER:
  794. return () => result.push(null);
  795. case NULL_AND_TRUE_HEADER:
  796. return () => result.push(null, true);
  797. case NULL_AND_FALSE_HEADER:
  798. return () => result.push(null, false);
  799. case NULL_AND_I8_HEADER:
  800. return () => {
  801. if (currentIsBuffer) {
  802. result.push(
  803. null,
  804. /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
  805. );
  806. currentPosition += I8_SIZE;
  807. checkOverflow();
  808. } else {
  809. result.push(null, read(I8_SIZE).readInt8(0));
  810. }
  811. };
  812. case NULL_AND_I32_HEADER:
  813. return () => {
  814. result.push(null);
  815. if (isInCurrentBuffer(I32_SIZE)) {
  816. result.push(
  817. /** @type {Buffer} */ (currentBuffer).readInt32LE(
  818. currentPosition
  819. )
  820. );
  821. currentPosition += I32_SIZE;
  822. checkOverflow();
  823. } else {
  824. result.push(read(I32_SIZE).readInt32LE(0));
  825. }
  826. };
  827. case NULLS8_HEADER:
  828. return () => {
  829. const len = readU8() + 4;
  830. for (let i = 0; i < len; i++) {
  831. result.push(null);
  832. }
  833. };
  834. case NULLS32_HEADER:
  835. return () => {
  836. const len = readU32() + 260;
  837. for (let i = 0; i < len; i++) {
  838. result.push(null);
  839. }
  840. };
  841. case BOOLEANS_HEADER:
  842. return () => {
  843. const innerHeader = readU8();
  844. if ((innerHeader & 0xf0) === 0) {
  845. readBits(innerHeader, 3);
  846. } else if ((innerHeader & 0xe0) === 0) {
  847. readBits(innerHeader, 4);
  848. } else if ((innerHeader & 0xc0) === 0) {
  849. readBits(innerHeader, 5);
  850. } else if ((innerHeader & 0x80) === 0) {
  851. readBits(innerHeader, 6);
  852. } else if (innerHeader !== 0xff) {
  853. let count = (innerHeader & 0x7f) + 7;
  854. while (count > 8) {
  855. readBits(readU8(), 8);
  856. count -= 8;
  857. }
  858. readBits(readU8(), count);
  859. } else {
  860. let count = readU32();
  861. while (count > 8) {
  862. readBits(readU8(), 8);
  863. count -= 8;
  864. }
  865. readBits(readU8(), count);
  866. }
  867. };
  868. case STRING_HEADER:
  869. return () => {
  870. const len = readU32();
  871. if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
  872. result.push(
  873. /** @type {Buffer} */
  874. (currentBuffer).toString(
  875. undefined,
  876. currentPosition,
  877. currentPosition + len
  878. )
  879. );
  880. currentPosition += len;
  881. checkOverflow();
  882. } else {
  883. result.push(read(len).toString());
  884. }
  885. };
  886. case SHORT_STRING_HEADER:
  887. return () => result.push("");
  888. case SHORT_STRING_HEADER | 1:
  889. return () => {
  890. if (currentIsBuffer && currentPosition < 0x7ffffffe) {
  891. result.push(
  892. /** @type {Buffer} */
  893. (currentBuffer).toString(
  894. "latin1",
  895. currentPosition,
  896. currentPosition + 1
  897. )
  898. );
  899. currentPosition++;
  900. checkOverflow();
  901. } else {
  902. result.push(read(1).toString("latin1"));
  903. }
  904. };
  905. case I8_HEADER:
  906. return () => {
  907. if (currentIsBuffer) {
  908. result.push(
  909. /** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
  910. );
  911. currentPosition++;
  912. checkOverflow();
  913. } else {
  914. result.push(read(1).readInt8(0));
  915. }
  916. };
  917. case BIGINT_I8_HEADER: {
  918. const len = 1;
  919. return () => {
  920. const need = I8_SIZE * len;
  921. if (isInCurrentBuffer(need)) {
  922. for (let i = 0; i < len; i++) {
  923. const value =
  924. /** @type {Buffer} */
  925. (currentBuffer).readInt8(currentPosition);
  926. result.push(BigInt(value));
  927. currentPosition += I8_SIZE;
  928. }
  929. checkOverflow();
  930. } else {
  931. const buf = read(need);
  932. for (let i = 0; i < len; i++) {
  933. const value = buf.readInt8(i * I8_SIZE);
  934. result.push(BigInt(value));
  935. }
  936. }
  937. };
  938. }
  939. case BIGINT_I32_HEADER: {
  940. const len = 1;
  941. return () => {
  942. const need = I32_SIZE * len;
  943. if (isInCurrentBuffer(need)) {
  944. for (let i = 0; i < len; i++) {
  945. const value = /** @type {Buffer} */ (currentBuffer).readInt32LE(
  946. currentPosition
  947. );
  948. result.push(BigInt(value));
  949. currentPosition += I32_SIZE;
  950. }
  951. checkOverflow();
  952. } else {
  953. const buf = read(need);
  954. for (let i = 0; i < len; i++) {
  955. const value = buf.readInt32LE(i * I32_SIZE);
  956. result.push(BigInt(value));
  957. }
  958. }
  959. };
  960. }
  961. case BIGINT_HEADER: {
  962. return () => {
  963. const len = readU32();
  964. if (isInCurrentBuffer(len) && currentPosition + len < 0x7fffffff) {
  965. const value =
  966. /** @type {Buffer} */
  967. (currentBuffer).toString(
  968. undefined,
  969. currentPosition,
  970. currentPosition + len
  971. );
  972. result.push(BigInt(value));
  973. currentPosition += len;
  974. checkOverflow();
  975. } else {
  976. const value = read(len).toString();
  977. result.push(BigInt(value));
  978. }
  979. };
  980. }
  981. default:
  982. if (header <= 10) {
  983. return () => result.push(header);
  984. } else if ((header & SHORT_STRING_HEADER) === SHORT_STRING_HEADER) {
  985. const len = header & SHORT_STRING_LENGTH_MASK;
  986. return () => {
  987. if (
  988. isInCurrentBuffer(len) &&
  989. currentPosition + len < 0x7fffffff
  990. ) {
  991. result.push(
  992. /** @type {Buffer} */
  993. (currentBuffer).toString(
  994. "latin1",
  995. currentPosition,
  996. currentPosition + len
  997. )
  998. );
  999. currentPosition += len;
  1000. checkOverflow();
  1001. } else {
  1002. result.push(read(len).toString("latin1"));
  1003. }
  1004. };
  1005. } else if ((header & NUMBERS_HEADER_MASK) === F64_HEADER) {
  1006. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1007. return () => {
  1008. const need = F64_SIZE * len;
  1009. if (isInCurrentBuffer(need)) {
  1010. for (let i = 0; i < len; i++) {
  1011. result.push(
  1012. /** @type {Buffer} */ (currentBuffer).readDoubleLE(
  1013. currentPosition
  1014. )
  1015. );
  1016. currentPosition += F64_SIZE;
  1017. }
  1018. checkOverflow();
  1019. } else {
  1020. const buf = read(need);
  1021. for (let i = 0; i < len; i++) {
  1022. result.push(buf.readDoubleLE(i * F64_SIZE));
  1023. }
  1024. }
  1025. };
  1026. } else if ((header & NUMBERS_HEADER_MASK) === I32_HEADER) {
  1027. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1028. return () => {
  1029. const need = I32_SIZE * len;
  1030. if (isInCurrentBuffer(need)) {
  1031. for (let i = 0; i < len; i++) {
  1032. result.push(
  1033. /** @type {Buffer} */ (currentBuffer).readInt32LE(
  1034. currentPosition
  1035. )
  1036. );
  1037. currentPosition += I32_SIZE;
  1038. }
  1039. checkOverflow();
  1040. } else {
  1041. const buf = read(need);
  1042. for (let i = 0; i < len; i++) {
  1043. result.push(buf.readInt32LE(i * I32_SIZE));
  1044. }
  1045. }
  1046. };
  1047. } else if ((header & NUMBERS_HEADER_MASK) === I8_HEADER) {
  1048. const len = (header & NUMBERS_COUNT_MASK) + 1;
  1049. return () => {
  1050. const need = I8_SIZE * len;
  1051. if (isInCurrentBuffer(need)) {
  1052. for (let i = 0; i < len; i++) {
  1053. result.push(
  1054. /** @type {Buffer} */ (currentBuffer).readInt8(
  1055. currentPosition
  1056. )
  1057. );
  1058. currentPosition += I8_SIZE;
  1059. }
  1060. checkOverflow();
  1061. } else {
  1062. const buf = read(need);
  1063. for (let i = 0; i < len; i++) {
  1064. result.push(buf.readInt8(i * I8_SIZE));
  1065. }
  1066. }
  1067. };
  1068. }
  1069. return () => {
  1070. throw new Error(`Unexpected header byte 0x${header.toString(16)}`);
  1071. };
  1072. }
  1073. });
  1074. /** @type {DeserializedType} */
  1075. let result = [];
  1076. while (currentBuffer !== null) {
  1077. if (typeof currentBuffer === "function") {
  1078. result.push(this._deserializeLazy(currentBuffer, context));
  1079. currentDataItem++;
  1080. currentBuffer =
  1081. currentDataItem < data.length ? data[currentDataItem] : null;
  1082. currentIsBuffer = Buffer.isBuffer(currentBuffer);
  1083. } else {
  1084. const header = readU8();
  1085. dispatchTable[header]();
  1086. }
  1087. }
  1088. // avoid leaking memory in context
  1089. // eslint-disable-next-line prefer-const
  1090. let _result = result;
  1091. result = /** @type {EXPECTED_ANY} */ (undefined);
  1092. return _result;
  1093. }
  1094. }
  1095. module.exports = BinaryMiddleware;
  1096. module.exports.MEASURE_START_OPERATION = MEASURE_START_OPERATION;
  1097. module.exports.MEASURE_END_OPERATION = MEASURE_END_OPERATION;