RealContentHashPlugin.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { SyncBailHook } = require("tapable");
  7. const { RawSource, CachedSource, CompatSource } = require("webpack-sources");
  8. const Compilation = require("../Compilation");
  9. const WebpackError = require("../WebpackError");
  10. const { compareSelect, compareStrings } = require("../util/comparators");
  11. const createHash = require("../util/createHash");
  12. /** @typedef {import("webpack-sources").Source} Source */
  13. /** @typedef {import("../Cache").Etag} Etag */
  14. /** @typedef {import("../Compilation").AssetInfo} AssetInfo */
  15. /** @typedef {import("../Compiler")} Compiler */
  16. /** @typedef {typeof import("../util/Hash")} Hash */
  17. const EMPTY_SET = new Set();
  18. /**
  19. * @template T
  20. * @param {T | T[]} itemOrItems item or items
  21. * @param {Set<T>} list list
  22. */
  23. const addToList = (itemOrItems, list) => {
  24. if (Array.isArray(itemOrItems)) {
  25. for (const item of itemOrItems) {
  26. list.add(item);
  27. }
  28. } else if (itemOrItems) {
  29. list.add(itemOrItems);
  30. }
  31. };
  32. /**
  33. * @template T
  34. * @param {T[]} input list
  35. * @param {function(T): Buffer} fn map function
  36. * @returns {Buffer[]} buffers without duplicates
  37. */
  38. const mapAndDeduplicateBuffers = (input, fn) => {
  39. // Buffer.equals compares size first so this should be efficient enough
  40. // If it becomes a performance problem we can use a map and group by size
  41. // instead of looping over all assets.
  42. const result = [];
  43. outer: for (const value of input) {
  44. const buf = fn(value);
  45. for (const other of result) {
  46. if (buf.equals(other)) continue outer;
  47. }
  48. result.push(buf);
  49. }
  50. return result;
  51. };
  52. /**
  53. * Escapes regular expression metacharacters
  54. * @param {string} str String to quote
  55. * @returns {string} Escaped string
  56. */
  57. const quoteMeta = str => str.replace(/[-[\]\\/{}()*+?.^$|]/g, "\\$&");
  58. const cachedSourceMap = new WeakMap();
  59. /**
  60. * @param {Source} source source
  61. * @returns {CachedSource} cached source
  62. */
  63. const toCachedSource = source => {
  64. if (source instanceof CachedSource) {
  65. return source;
  66. }
  67. const entry = cachedSourceMap.get(source);
  68. if (entry !== undefined) return entry;
  69. const newSource = new CachedSource(CompatSource.from(source));
  70. cachedSourceMap.set(source, newSource);
  71. return newSource;
  72. };
  73. /** @typedef {Set<string>} OwnHashes */
  74. /** @typedef {Set<string>} ReferencedHashes */
  75. /** @typedef {Set<string>} Hashes */
  76. /**
  77. * @typedef {object} AssetInfoForRealContentHash
  78. * @property {string} name
  79. * @property {AssetInfo} info
  80. * @property {Source} source
  81. * @property {RawSource | undefined} newSource
  82. * @property {RawSource | undefined} newSourceWithoutOwn
  83. * @property {string} content
  84. * @property {OwnHashes | undefined} ownHashes
  85. * @property {Promise<void> | undefined} contentComputePromise
  86. * @property {Promise<void> | undefined} contentComputeWithoutOwnPromise
  87. * @property {ReferencedHashes | undefined} referencedHashes
  88. * @property {Hashes} hashes
  89. */
  90. /**
  91. * @typedef {object} CompilationHooks
  92. * @property {SyncBailHook<[Buffer[], string], string | void>} updateHash
  93. */
  94. /** @type {WeakMap<Compilation, CompilationHooks>} */
  95. const compilationHooksMap = new WeakMap();
  96. class RealContentHashPlugin {
  97. /**
  98. * @param {Compilation} compilation the compilation
  99. * @returns {CompilationHooks} the attached hooks
  100. */
  101. static getCompilationHooks(compilation) {
  102. if (!(compilation instanceof Compilation)) {
  103. throw new TypeError(
  104. "The 'compilation' argument must be an instance of Compilation"
  105. );
  106. }
  107. let hooks = compilationHooksMap.get(compilation);
  108. if (hooks === undefined) {
  109. hooks = {
  110. updateHash: new SyncBailHook(["content", "oldHash"])
  111. };
  112. compilationHooksMap.set(compilation, hooks);
  113. }
  114. return hooks;
  115. }
  116. /**
  117. * @param {object} options options object
  118. * @param {string | Hash} options.hashFunction the hash function to use
  119. * @param {string} options.hashDigest the hash digest to use
  120. */
  121. constructor({ hashFunction, hashDigest }) {
  122. this._hashFunction = hashFunction;
  123. this._hashDigest = hashDigest;
  124. }
  125. /**
  126. * Apply the plugin
  127. * @param {Compiler} compiler the compiler instance
  128. * @returns {void}
  129. */
  130. apply(compiler) {
  131. compiler.hooks.compilation.tap("RealContentHashPlugin", compilation => {
  132. const cacheAnalyse = compilation.getCache(
  133. "RealContentHashPlugin|analyse"
  134. );
  135. const cacheGenerate = compilation.getCache(
  136. "RealContentHashPlugin|generate"
  137. );
  138. const hooks = RealContentHashPlugin.getCompilationHooks(compilation);
  139. compilation.hooks.processAssets.tapPromise(
  140. {
  141. name: "RealContentHashPlugin",
  142. stage: Compilation.PROCESS_ASSETS_STAGE_OPTIMIZE_HASH
  143. },
  144. async () => {
  145. const assets = compilation.getAssets();
  146. /** @type {AssetInfoForRealContentHash[]} */
  147. const assetsWithInfo = [];
  148. /** @type {Map<string, [AssetInfoForRealContentHash]>} */
  149. const hashToAssets = new Map();
  150. for (const { source, info, name } of assets) {
  151. const cachedSource = toCachedSource(source);
  152. const content = /** @type {string} */ (cachedSource.source());
  153. /** @type {Hashes} */
  154. const hashes = new Set();
  155. addToList(info.contenthash, hashes);
  156. /** @type {AssetInfoForRealContentHash} */
  157. const data = {
  158. name,
  159. info,
  160. source: cachedSource,
  161. newSource: undefined,
  162. newSourceWithoutOwn: undefined,
  163. content,
  164. ownHashes: undefined,
  165. contentComputePromise: undefined,
  166. contentComputeWithoutOwnPromise: undefined,
  167. referencedHashes: undefined,
  168. hashes
  169. };
  170. assetsWithInfo.push(data);
  171. for (const hash of hashes) {
  172. const list = hashToAssets.get(hash);
  173. if (list === undefined) {
  174. hashToAssets.set(hash, [data]);
  175. } else {
  176. list.push(data);
  177. }
  178. }
  179. }
  180. if (hashToAssets.size === 0) return;
  181. const hashRegExp = new RegExp(
  182. Array.from(hashToAssets.keys(), quoteMeta).join("|"),
  183. "g"
  184. );
  185. await Promise.all(
  186. assetsWithInfo.map(async asset => {
  187. const { name, source, content, hashes } = asset;
  188. if (Buffer.isBuffer(content)) {
  189. asset.referencedHashes = EMPTY_SET;
  190. asset.ownHashes = EMPTY_SET;
  191. return;
  192. }
  193. const etag = cacheAnalyse.mergeEtags(
  194. cacheAnalyse.getLazyHashedEtag(source),
  195. Array.from(hashes).join("|")
  196. );
  197. [asset.referencedHashes, asset.ownHashes] =
  198. await cacheAnalyse.providePromise(name, etag, () => {
  199. const referencedHashes = new Set();
  200. const ownHashes = new Set();
  201. const inContent = content.match(hashRegExp);
  202. if (inContent) {
  203. for (const hash of inContent) {
  204. if (hashes.has(hash)) {
  205. ownHashes.add(hash);
  206. continue;
  207. }
  208. referencedHashes.add(hash);
  209. }
  210. }
  211. return [referencedHashes, ownHashes];
  212. });
  213. })
  214. );
  215. /**
  216. * @param {string} hash the hash
  217. * @returns {undefined | ReferencedHashes} the referenced hashes
  218. */
  219. const getDependencies = hash => {
  220. const assets = hashToAssets.get(hash);
  221. if (!assets) {
  222. const referencingAssets = assetsWithInfo.filter(asset =>
  223. /** @type {ReferencedHashes} */ (asset.referencedHashes).has(
  224. hash
  225. )
  226. );
  227. const err = new WebpackError(`RealContentHashPlugin
  228. Some kind of unexpected caching problem occurred.
  229. An asset was cached with a reference to another asset (${hash}) that's not in the compilation anymore.
  230. Either the asset was incorrectly cached, or the referenced asset should also be restored from cache.
  231. Referenced by:
  232. ${referencingAssets
  233. .map(a => {
  234. const match = new RegExp(`.{0,20}${quoteMeta(hash)}.{0,20}`).exec(
  235. a.content
  236. );
  237. return ` - ${a.name}: ...${match ? match[0] : "???"}...`;
  238. })
  239. .join("\n")}`);
  240. compilation.errors.push(err);
  241. return;
  242. }
  243. const hashes = new Set();
  244. for (const { referencedHashes, ownHashes } of assets) {
  245. if (!(/** @type {OwnHashes} */ (ownHashes).has(hash))) {
  246. for (const hash of /** @type {OwnHashes} */ (ownHashes)) {
  247. hashes.add(hash);
  248. }
  249. }
  250. for (const hash of /** @type {ReferencedHashes} */ (
  251. referencedHashes
  252. )) {
  253. hashes.add(hash);
  254. }
  255. }
  256. return hashes;
  257. };
  258. /**
  259. * @param {string} hash the hash
  260. * @returns {string} the hash info
  261. */
  262. const hashInfo = hash => {
  263. const assets = hashToAssets.get(hash);
  264. return `${hash} (${Array.from(
  265. /** @type {AssetInfoForRealContentHash[]} */ (assets),
  266. a => a.name
  267. )})`;
  268. };
  269. const hashesInOrder = new Set();
  270. for (const hash of hashToAssets.keys()) {
  271. /**
  272. * @param {string} hash the hash
  273. * @param {Set<string>} stack stack of hashes
  274. */
  275. const add = (hash, stack) => {
  276. const deps = getDependencies(hash);
  277. if (!deps) return;
  278. stack.add(hash);
  279. for (const dep of deps) {
  280. if (hashesInOrder.has(dep)) continue;
  281. if (stack.has(dep)) {
  282. throw new Error(
  283. `Circular hash dependency ${Array.from(
  284. stack,
  285. hashInfo
  286. ).join(" -> ")} -> ${hashInfo(dep)}`
  287. );
  288. }
  289. add(dep, stack);
  290. }
  291. hashesInOrder.add(hash);
  292. stack.delete(hash);
  293. };
  294. if (hashesInOrder.has(hash)) continue;
  295. add(hash, new Set());
  296. }
  297. const hashToNewHash = new Map();
  298. /**
  299. * @param {AssetInfoForRealContentHash} asset asset info
  300. * @returns {Etag} etag
  301. */
  302. const getEtag = asset =>
  303. cacheGenerate.mergeEtags(
  304. cacheGenerate.getLazyHashedEtag(asset.source),
  305. Array.from(
  306. /** @type {ReferencedHashes} */ (asset.referencedHashes),
  307. hash => hashToNewHash.get(hash)
  308. ).join("|")
  309. );
  310. /**
  311. * @param {AssetInfoForRealContentHash} asset asset info
  312. * @returns {Promise<void>}
  313. */
  314. const computeNewContent = asset => {
  315. if (asset.contentComputePromise) return asset.contentComputePromise;
  316. return (asset.contentComputePromise = (async () => {
  317. if (
  318. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  319. Array.from(
  320. /** @type {ReferencedHashes} */
  321. (asset.referencedHashes)
  322. ).some(hash => hashToNewHash.get(hash) !== hash)
  323. ) {
  324. const identifier = asset.name;
  325. const etag = getEtag(asset);
  326. asset.newSource = await cacheGenerate.providePromise(
  327. identifier,
  328. etag,
  329. () => {
  330. const newContent = asset.content.replace(hashRegExp, hash =>
  331. hashToNewHash.get(hash)
  332. );
  333. return new RawSource(newContent);
  334. }
  335. );
  336. }
  337. })());
  338. };
  339. /**
  340. * @param {AssetInfoForRealContentHash} asset asset info
  341. * @returns {Promise<void>}
  342. */
  343. const computeNewContentWithoutOwn = asset => {
  344. if (asset.contentComputeWithoutOwnPromise)
  345. return asset.contentComputeWithoutOwnPromise;
  346. return (asset.contentComputeWithoutOwnPromise = (async () => {
  347. if (
  348. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  349. Array.from(
  350. /** @type {ReferencedHashes} */
  351. (asset.referencedHashes)
  352. ).some(hash => hashToNewHash.get(hash) !== hash)
  353. ) {
  354. const identifier = `${asset.name}|without-own`;
  355. const etag = getEtag(asset);
  356. asset.newSourceWithoutOwn = await cacheGenerate.providePromise(
  357. identifier,
  358. etag,
  359. () => {
  360. const newContent = asset.content.replace(
  361. hashRegExp,
  362. hash => {
  363. if (
  364. /** @type {OwnHashes} */ (asset.ownHashes).has(hash)
  365. ) {
  366. return "";
  367. }
  368. return hashToNewHash.get(hash);
  369. }
  370. );
  371. return new RawSource(newContent);
  372. }
  373. );
  374. }
  375. })());
  376. };
  377. const comparator = compareSelect(a => a.name, compareStrings);
  378. for (const oldHash of hashesInOrder) {
  379. const assets =
  380. /** @type {AssetInfoForRealContentHash[]} */
  381. (hashToAssets.get(oldHash));
  382. assets.sort(comparator);
  383. await Promise.all(
  384. assets.map(asset =>
  385. /** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)
  386. ? computeNewContentWithoutOwn(asset)
  387. : computeNewContent(asset)
  388. )
  389. );
  390. const assetsContent = mapAndDeduplicateBuffers(assets, asset => {
  391. if (/** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)) {
  392. return asset.newSourceWithoutOwn
  393. ? asset.newSourceWithoutOwn.buffer()
  394. : asset.source.buffer();
  395. }
  396. return asset.newSource
  397. ? asset.newSource.buffer()
  398. : asset.source.buffer();
  399. });
  400. let newHash = hooks.updateHash.call(assetsContent, oldHash);
  401. if (!newHash) {
  402. const hash = createHash(this._hashFunction);
  403. if (compilation.outputOptions.hashSalt) {
  404. hash.update(compilation.outputOptions.hashSalt);
  405. }
  406. for (const content of assetsContent) {
  407. hash.update(content);
  408. }
  409. const digest = hash.digest(this._hashDigest);
  410. newHash = /** @type {string} */ (digest.slice(0, oldHash.length));
  411. }
  412. hashToNewHash.set(oldHash, newHash);
  413. }
  414. await Promise.all(
  415. assetsWithInfo.map(async asset => {
  416. await computeNewContent(asset);
  417. const newName = asset.name.replace(hashRegExp, hash =>
  418. hashToNewHash.get(hash)
  419. );
  420. const infoUpdate = {};
  421. const hash = asset.info.contenthash;
  422. infoUpdate.contenthash = Array.isArray(hash)
  423. ? hash.map(hash => hashToNewHash.get(hash))
  424. : hashToNewHash.get(hash);
  425. if (asset.newSource !== undefined) {
  426. compilation.updateAsset(
  427. asset.name,
  428. asset.newSource,
  429. infoUpdate
  430. );
  431. } else {
  432. compilation.updateAsset(asset.name, asset.source, infoUpdate);
  433. }
  434. if (asset.name !== newName) {
  435. compilation.renameAsset(asset.name, newName);
  436. }
  437. })
  438. );
  439. }
  440. );
  441. });
  442. }
  443. }
  444. module.exports = RealContentHashPlugin;