Today, let's implement a simple packaging tool

File dependency

src
├─ a.js
├─ b.js
├─ c.js
└─ index.js

[external chain picture transfer failed. The source station may have anti-theft chain mechanism. It is recommended to save the picture and upload it directly (img-xxkfcw02-1643883138652)( https://raw.githubusercontent.com/nxl3477/md-img-storage/study/2022/202201271747838.png )]

The contents of the document are as follows

// src/index.js
import { aInit } from './a.js'

aInit()

// src/a.js
import b from './b.js'
import c from './c.js'
export const aInit = () => {
  console.log('a init')
}

// src/b.js
console.log('b import')

// src/c.js
console.log('c import')

thinking

The core principle is three stages: analysis transformation generation

analysis

What is resolved is the dependency of the file, which is collected for subsequent conversion

transformation

1. Convert es module import syntax to commonjs import syntax
2. Modularization using iife

generate

Output the converted code to bundle JS file

summary

With the three macro processes, we can think about how to implement this packaging tool:

1. Read file contents
2. Analyze the contents of the file to find its sub dependencies
3. Save the code and sub dependency list of the current file, and if there are sub dependencies, return to the first step to process the sub dependencies one by one, and generate the dependency diagram after all processing
4. Convert es module to common js to be compatible with browsers
5. Generate iife code for modularization
6. The output is bundle js

Start doing it

First, we create a mini webpack. Net in the root directory JS file to write our mini webpack

1. Read file contents

Use the fs module, pass in the entry file address, and export the file path and read content

const fs = require('fs');

function createAsset(filePath) {

	const originCode = fs.readFileSync(filePath, 'utf8')
	
	return {
	
			filePath,
			
			code: originCode,
		
		}

}

createAsset('./src/index.js')

2. Analyze the contents of the file and find out its sub dependencies

How can we know the dependency of a file? There are generally two schemes

1. The disadvantage of using regular to match keywords such as import from is that it is relatively inflexible
2. Using ast

Of course, we use ast here. When it comes to ast, babel is a good thing. Students who are not familiar with babel can read this article written earlier: Understand all the source map s of webpack in one article! 🤔

Use @ babel/parser to generate the ast, and then use @ babel/traverse to process the ast. Add the analyzed dependencies into the deps array and export them

const parser = require('@babel/parser')
const traverse = require('@babel/traverse')
function createAsset(filePath) {
	const originCode = fs.readFileSync(filePath, 'utf8')
	
	const ast = parser.parse(originCode, {
		// It needs to be declared as es module 
		sourceType: 'module'	
	});

	// Internal import dependencies collected
	const deps = []
	traverse.default(ast, {
		// Find import 
		ImportDeclaration(path) {
			const node = path.node
			// Dependency array collected
			deps.push(node.source.value)
		}
	})

	return {
		filePath,
		code,
		deps
	}
}

3. Save the code and sub dependency list of the current file, and if there are sub dependencies, go back to the first step to deal with them one by one

Using breadth first traversal and taking the entry as the starting point, collect the dependencies of all resources into the graph

function createGraph (entryFile) {
  const entryAsset = createAsset(entryFile)
  // The first is to add the entry file
  const graph = [ entryAsset ]
  
  // Traverse all resources in the graph
  for (const asset of graph) {
    // If the current resource has sub dependencies
    asset.deps.forEach(relativePath => {
      const childAsset = createAsset(path.join(__dirname, 'src', relativePath))
	  // Add sub dependent resources to the graph array, and subsequent for loops will naturally process them
      graph.push(childAsset)
    })
  }

  return graph
}
// call
const graph = createGraph('./src/index.js')

Let's take a look at the output of the graph at this time
[the external chain image transfer fails. The source station may have an anti-theft chain mechanism. It is recommended to save the image and upload it directly (img-jbfzootg-1643883138653)( https://raw.githubusercontent.com/nxl3477/md-img-storage/study/2022/202201271842498.png )]

We got the file path, file content and sub dependencies of each file

4. Convert es module to common js to be compatible with the browser

babel can also do this conversion. At this time, we modify the createAsset method to use the @ babel/core
transformFromAst: use @ Babel / preset env to convert ast back to code

const { transformFromAst } =require("@babel/core")

function createAsset(filePath) {
  const originCode = fs.readFileSync(filePath, 'utf8')

  const ast = parser.parse(originCode, {
    sourceType: 'module'
  });

  // Internal import dependencies collected
  const deps = []
  traverse.default(ast, {
    ImportDeclaration(path) {
      const node = path.node
      // All detected dependencies are added to the array
      deps.push(node.source.value)
    }
  })


  // Convert es module to common js to be compatible with browsers
  const { code } = transformFromAst(ast, originCode, {
    "presets": ["@babel/preset-env"]
  })


  return {
    filePath,
    code,
    deps
  }
}

Let's take another look at the output of graph

You can see that the code inside is no longer the import in the previous stage, but is imported using require

5. Generate iife code to realize modularization

Although we have changed the dependency of files and converted es module into common js, we still can't use it in the browser. Therefore, we want to realize iife modularization. Let's see how to write it

First of all, our purpose is to put these files in the final bundle js

So let's manually put all the file contents into the bundle JS look

// index.js
import { aInit } from './a.js'
aInit()

// a.js
import b from './b.js'
import c from './c.js'
export const aInit = () => {
  console.log('a init')
}

// b.js
console.log('b import')

// c.js
console.log('c import')

Obviously, this can not be put together directly
First, you need to manually change esmodule to common js

// index.js
const { aInit } = require('./a.js') 
aInit()

// a.js
const b = require('./b.js') 
const c = require('./c.js') 
module.exports.aInit = () => {
  console.log('a init')
}

// b.js
console.log('b import')

// c.js
console.log('c import')

After changing to require, it also cannot run because there is no require method in the browser, so we need to give it a

// index.js
function indexJs(require, module, exports) {
  const { aInit } = require('./a.js') 
  aInit()
}

// a.js
function aJs(require, module, exports) {
  const b = require('./b.js') 
  const c = require('./c.js') 
  module.exports.aInit = () => {
    console.log('a init')
  }
}
// b.js
function bJs(require, module, exports) {
  console.log('b import')
}

// c.js
function cJs(require, module, exports) {
  console.log('c import')
}

Each file is regarded as a function, which will receive three parameters: require, module and exports. Now there is a place to take the required, but we haven't done a specific implementation yet, so we need to continue the transformation.

(function (moduleMap) {
  function require(filePath) {
    const fn = moduleMap[filePath]

    const module = {
      exports: {}
    }
    fn(require, module, module.exports)
    return module.exports
  }

  require('./index.js')
}({
  './index.js': function (require, module, exports) {
    const { aInit } = require('./a.js')
    aInit()
  },
  './a.js': function (require, module, exports) {
    const b = require('./b.js') 
    const c = require('./c.js') 
    module.exports.aInit = () => {
      console.log('a init')
    }
  },
  './b.js': function (require, module, exports) {
    console.log('b import')
  },
  './c.js': function (require, module, exports) {
    console.log('c import')
  }
}))

Use iife to wrap, pass in the file map as a parameter, and construct the require method in the function. Its internal essence is to obtain the corresponding function from the file map, construct a module object, call function and pass it in

And finally, call the entrance file.

Now let's load this code in the browser and see if it can run
[external chain picture transfer failed. The source station may have anti-theft chain mechanism. It is recommended to save the picture and upload it directly (img-ubfzv3ku-1643883138655)( https://raw.githubusercontent.com/nxl3477/md-img-storage/study/2022/202201271916623.png )]

ok, no problem

6. The output is bundle js

How to dynamically construct iife? A more convenient way is to use template engine

We write a build method using bundle Rendering with EJS template

function build(graph) {
  const template = fs.readFileSync('./template/bundle.ejs', { encoding: 'utf-8' })
  
  const ejsData = graph.map(asset => ({
    filePath: asset.filePath,
    code: asset.code
  }))
  console.log(ejsData)

  const code = ejs.render(template, { ejsData })

  fs.writeFileSync('./dist/bundle.js', code)
}

const graph = createGraph('./src/index.js')


build(graph)

bundle. The content of ejs template is as follows [the specific syntax of ejs template engine can be viewed in official documents]

// bundle.ejs
(function (moduleMap) {
  function require(filePath) {
    const fn = moduleMap[filePath]

    const module = {
      exports: {}
    }
    fn(require, module, module.exports)
    return module.exports
  }

  require('./index.js')
}({
  <% ejsData.forEach(item => { %>
  "<%- item["filePath"] %>": function (require, module, exports) {
    <%- item["code"] %>
  },
  <% }) %>
}))

ejsData is the variable we need to inject, that is, our resource relationship dependency graph. forEach it and cycle out all resources

At this time, the final documents are as follows:

(function (moduleMap) {
  function require(filePath) {
    const fn = moduleMap[filePath];

    const module = {
      exports: {},
    };
    fn(require, module, module.exports);
    return module.exports;
  }

  require("./index.js");
})({
  "./src/index.js": function (require, module, exports) {
    "use strict";

    var _a = require("./a.js");

    (0, _a.aInit)();
  },

  "/Users/nxl3477/Documents/study_space/mini-series/mini-webpack/my-mini-webpack/src/a.js":
    function (require, module, exports) {
      "use strict";

      Object.defineProperty(exports, "__esModule", {
        value: true,
      });
      exports.aInit = void 0;

      var _b = _interopRequireDefault(require("./b.js"));

      var _c = _interopRequireDefault(require("./c.js"));

      function _interopRequireDefault(obj) {
        return obj && obj.__esModule ? obj : { default: obj };
      }

      var aInit = function aInit() {
        console.log("a init");
      };

      exports.aInit = aInit;
    },

  "/Users/nxl3477/Documents/study_space/mini-series/mini-webpack/my-mini-webpack/src/b.js":
    function (require, module, exports) {
      "use strict";

      console.log("b import");
    },

  "/Users/nxl3477/Documents/study_space/mini-series/mini-webpack/my-mini-webpack/src/c.js":
    function (require, module, exports) {
      "use strict";

      console.log("c import");
    },
});

We found some problems with the output documents,
What we actually need is the relative path of the file, and many of the resources of the incoming moduleMap are absolute paths. Therefore, for example, we can't find the require('. / a.js'), so we have to find a way to transform it

The iife code is modified as follows. We give all resource modules an id and carry an object for relative path mapping to the actual id

(function (moduleMapping) {
  function require(moduleId) {
	// Fetch function and mapping object
    const [ fn, mapping ] = moduleMapping[moduleId]

	// Wrap a layer of require, first get the actual id through its own mapping, and then call require to get the actual resource
    function localRequre(filePath) {
      const _moduleId = mapping[filePath]
      return require(_moduleId)
    }

    const module = {
      exports: {}
    }

    fn(localRequre, module, module.exports)
    return module.exports
  }

  require(0)
}({
  // The first element is a function, and the second is an object used to map relative paths
  // ./index.js
  0: [function (require, module, exports) {
    const { aInit } = require('./a.js')
    aInit()
  }, { './a.js': 1 }],
  // ./a.js
  1: [function (require, module, exports) {
    const b = require('./b.js') 
    const c = require('./c.js') 
    module.exports.aInit = () => {
      console.log('a init')
    }
  }, { './b.js': 2, './c.js': 3 }],
  // './b.js'
  2: [function (require, module, exports) {
    console.log('b import')
  }, {}],
  // './c.js'
  3: [function (require, module, exports) {
    console.log('c import')
  }, {}]
}))

According to the above code, let's update the ejs template

// bundle.ejs
(function (moduleMapping) {
  function require(moduleId) {
    const [ fn, mapping ] = moduleMapping[moduleId]

    function localRequre(filePath) {
      const _moduleId = mapping[filePath]
      return require(_moduleId)
    }

    const module = {
      exports: {}
    }

    fn(localRequre, module, module.exports)
    return module.exports
  }
  // Entry Id is 0
  require(0)
}({
  <% ejsData.forEach(item => { %>
  "<%- item["id"] %>": [function (require, module, exports) {
    <%- item["code"] %>
  }, <%- JSON.stringify(item.mapping) %> ],
  <% }) %>
}))

mini-webpack.js also needs to be adjusted:

1. Generate an id for each resource module
2. mapping is constructed for each resource module to map the module id of sub dependencies

let uid = 0

function createAsset(filePath) {
  const originCode = fs.readFileSync(filePath, 'utf8')

  const ast = parser.parse(originCode, {
    sourceType: 'module'
  });

  // Internal import dependencies collected
  const deps = []
  traverse.default(ast, {
    ImportDeclaration(path) {
      const node = path.node
      // All detected dependencies are added to the array
      deps.push(node.source.value)
    }
  })

  const { code } = transformFromAst(ast, originCode, {
    "presets": ["@babel/preset-env"]
  })

  return {
    filePath,
    code,
    // Add mapping
    mapping: {},
    deps,
	// Add id
    id: uid++
  }
}


// Improve mapping data
function createGraph (entryFile) {
  const entryAsset = createAsset(entryFile)
  const graph = [ entryAsset ]
  
  // Traverse all resources in the graph
  for (const asset of graph) {
    // Compile dependencies for each resource
    asset.deps.forEach(relativePath => {
      const child = createAsset(path.join(__dirname, 'src', relativePath))
      // Write the id of the child dependency to the id of the parent resource module
      asset.mapping[relativePath] = child.id
      graph.push(child)
    })
  }

  return graph
}


// Improve the data of ejsData
function build(graph) {
  const template = fs.readFileSync('./template/bundle.ejs', { encoding: 'utf-8' })
  
  const ejsData = graph.map(asset => ({
    filePath: asset.filePath,
    code: asset.code,
    id: asset.id,
    mapping: asset.mapping
  }))
  console.log(ejsData)

  const code = ejs.render(template, { ejsData })

  fs.writeFileSync('./dist/bundle.js', code)
}

Well, at this point, let's look at the generated code

(function (moduleMapping) {
  function require(moduleId) {
    const [fn, mapping] = moduleMapping[moduleId];

    function localRequre(filePath) {
      const _moduleId = mapping[filePath];
      return require(_moduleId);
    }

    const module = {
      exports: {},
    };

    fn(localRequre, module, module.exports);
    return module.exports;
  }

  require(0);
})({
  0: [
    function (require, module, exports) {
      "use strict";

      var _a = require("./a.js");

      (0, _a.aInit)();
    },
    { "./a.js": 1 },
  ],

  1: [
    function (require, module, exports) {
      "use strict";

      Object.defineProperty(exports, "__esModule", {
        value: true,
      });
      exports.aInit = void 0;

      var _b = _interopRequireDefault(require("./b.js"));

      var _c = _interopRequireDefault(require("./c.js"));

      function _interopRequireDefault(obj) {
        return obj && obj.__esModule ? obj : { default: obj };
      }

      var aInit = function aInit() {
        console.log("a init");
      };

      exports.aInit = aInit;
    },
    { "./b.js": 2, "./c.js": 3 },
  ],

  2: [
    function (require, module, exports) {
      "use strict";

      console.log("b import");
    },
    {},
  ],

  3: [
    function (require, module, exports) {
      "use strict";

      console.log("c import");
    },
    {},
  ],
});

Then go to the browser and execute it

ok, perfect, a simple packaging tool is completed

The complete code has been uploaded to github: mini-webpack

reference material:
It only takes 80 lines of code to understand the core of webpack