More rants on the dependencies issue
So today I needed to copy a file in a node based JavaScript build step.
Background: For those that don't know it node has a package manager called npm (Node Package
Manager). Packages have a package.json file that defines tons of things and
that includes a "scripts" section which are effectively just tiny command line strings
associated with a keyword.
Examples
"scripts": {
"build": "make -f makefile",
"test": "runtest-harness"
}So you can now type npm run build to run the build script and it will run just
as if you had typed make -f makefile.
Other than organizational the biggest plus is that if you have any development dependencies npm will look in those locally installed dependencies to run the commands. This means all your tools can be local to your project. If this project needs lint 1.6 and some other project needs lint 2.9 no worries. Just add the correct version of lint to your development dependencies and npm will run it for you.
But, the issue comes up, I wanted to copy a file. I could use a bigger build system
but for small things you can imagine just wanting to use cp as in
"scripts": {
"build": "make -f makefile && cp a.out dist/MyApp",
...The problem is cp is mac/linux only. If you care about Windows devs being able
to build on Windows then you can't use cp. The solution is to add a node based
copy command to your development dependencies and then you can use it cross
platform
So, I go looking for copy commands. One of the most popular is [cpy-cli].
Here's its dependency tree
└─┬ cpy-cli@2.0.0
├─┬ cpy@7.3.0
│ ├── arrify@1.0.1
│ ├─┬ cp-file@6.2.0
│ │ ├── graceful-fs@4.2.3
│ │ ├─┬ make-dir@2.1.0
│ │ │ ├── pify@4.0.1 deduped
│ │ │ └── semver@5.7.1 deduped
│ │ ├── nested-error-stacks@2.1.0 deduped
│ │ ├── pify@4.0.1
│ │ └── safe-buffer@5.2.0
│ ├─┬ globby@9.2.0
│ │ ├─┬ @types/glob@7.1.1
│ │ │ ├── @types/events@3.0.0
│ │ │ ├── @types/minimatch@3.0.3
│ │ │ └── @types/node@12.11.6
│ │ ├─┬ array-union@1.0.2
│ │ │ └── array-uniq@1.0.3
│ │ ├─┬ dir-glob@2.2.2
│ │ │ └─┬ path-type@3.0.0
│ │ │ └── pify@3.0.0
│ │ ├─┬ fast-glob@2.2.7
│ │ │ ├─┬ @mrmlnc/readdir-enhanced@2.2.1
│ │ │ │ ├── call-me-maybe@1.0.1
│ │ │ │ └── glob-to-regexp@0.3.0
│ │ │ ├── @nodelib/fs.stat@1.1.3
│ │ │ ├─┬ glob-parent@3.1.0
│ │ │ │ ├─┬ is-glob@3.1.0
│ │ │ │ │ └── is-extglob@2.1.1 deduped
│ │ │ │ └── path-dirname@1.0.2
│ │ │ ├─┬ is-glob@4.0.1
│ │ │ │ └── is-extglob@2.1.1
│ │ │ ├── merge2@1.3.0
│ │ │ └─┬ micromatch@3.1.10
│ │ │ ├── arr-diff@4.0.0
│ │ │ ├── array-unique@0.3.2
│ │ │ ├─┬ braces@2.3.2
│ │ │ │ ├── arr-flatten@1.1.0
│ │ │ │ ├── array-unique@0.3.2 deduped
│ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ └── is-extendable@0.1.1
│ │ │ │ ├─┬ fill-range@4.0.0
│ │ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ │ └── is-extendable@0.1.1 deduped
│ │ │ │ │ ├─┬ is-number@3.0.0
│ │ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ │ └── is-buffer@1.1.6
│ │ │ │ │ ├── repeat-string@1.6.1
│ │ │ │ │ └─┬ to-regex-range@2.1.1
│ │ │ │ │ ├── is-number@3.0.0 deduped
│ │ │ │ │ └── repeat-string@1.6.1 deduped
│ │ │ │ ├── isobject@3.0.1
│ │ │ │ ├── repeat-element@1.1.3
│ │ │ │ ├── snapdragon@0.8.2 deduped
│ │ │ │ ├─┬ snapdragon-node@2.1.1
│ │ │ │ │ ├─┬ define-property@1.0.0
│ │ │ │ │ │ └─┬ is-descriptor@1.0.2
│ │ │ │ │ │ ├─┬ is-accessor-descriptor@1.0.0
│ │ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ │ ├─┬ is-data-descriptor@1.0.0
│ │ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ ├── isobject@3.0.1 deduped
│ │ │ │ │ └─┬ snapdragon-util@3.0.1
│ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ ├─┬ split-string@3.1.0
│ │ │ │ │ └── extend-shallow@3.0.2 deduped
│ │ │ │ └── to-regex@3.0.2 deduped
│ │ │ ├─┬ define-property@2.0.2
│ │ │ │ ├─┬ is-descriptor@1.0.2
│ │ │ │ │ ├─┬ is-accessor-descriptor@1.0.0
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ ├─┬ is-data-descriptor@1.0.0
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ └── isobject@3.0.1 deduped
│ │ │ ├─┬ extend-shallow@3.0.2
│ │ │ │ ├── assign-symbols@1.0.0
│ │ │ │ └─┬ is-extendable@1.0.1
│ │ │ │ └─┬ is-plain-object@2.0.4
│ │ │ │ └── isobject@3.0.1 deduped
│ │ │ ├─┬ extglob@2.0.4
│ │ │ │ ├── array-unique@0.3.2 deduped
│ │ │ │ ├─┬ define-property@1.0.0
│ │ │ │ │ └─┬ is-descriptor@1.0.2
│ │ │ │ │ ├─┬ is-accessor-descriptor@1.0.0
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ ├─┬ is-data-descriptor@1.0.0
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ ├─┬ expand-brackets@2.1.4
│ │ │ │ │ ├── debug@2.6.9 deduped
│ │ │ │ │ ├─┬ define-property@0.2.5
│ │ │ │ │ │ └── is-descriptor@0.1.6 deduped
│ │ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ │ └── is-extendable@0.1.1 deduped
│ │ │ │ │ ├── posix-character-classes@0.1.1
│ │ │ │ │ ├── regex-not@1.0.2 deduped
│ │ │ │ │ ├── snapdragon@0.8.2 deduped
│ │ │ │ │ └── to-regex@3.0.2 deduped
│ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ └── is-extendable@0.1.1 deduped
│ │ │ │ ├── fragment-cache@0.2.1 deduped
│ │ │ │ ├── regex-not@1.0.2 deduped
│ │ │ │ ├── snapdragon@0.8.2 deduped
│ │ │ │ └── to-regex@3.0.2 deduped
│ │ │ ├─┬ fragment-cache@0.2.1
│ │ │ │ └── map-cache@0.2.2
│ │ │ ├── kind-of@6.0.2
│ │ │ ├─┬ nanomatch@1.2.13
│ │ │ │ ├── arr-diff@4.0.0 deduped
│ │ │ │ ├── array-unique@0.3.2 deduped
│ │ │ │ ├── define-property@2.0.2 deduped
│ │ │ │ ├── extend-shallow@3.0.2 deduped
│ │ │ │ ├── fragment-cache@0.2.1 deduped
│ │ │ │ ├── is-windows@1.0.2
│ │ │ │ ├── kind-of@6.0.2 deduped
│ │ │ │ ├── object.pick@1.3.0 deduped
│ │ │ │ ├── regex-not@1.0.2 deduped
│ │ │ │ ├── snapdragon@0.8.2 deduped
│ │ │ │ └── to-regex@3.0.2 deduped
│ │ │ ├─┬ object.pick@1.3.0
│ │ │ │ └── isobject@3.0.1 deduped
│ │ │ ├─┬ regex-not@1.0.2
│ │ │ │ ├── extend-shallow@3.0.2 deduped
│ │ │ │ └─┬ safe-regex@1.1.0
│ │ │ │ └── ret@0.1.15
│ │ │ ├─┬ snapdragon@0.8.2
│ │ │ │ ├─┬ base@0.11.2
│ │ │ │ │ ├─┬ cache-base@1.0.1
│ │ │ │ │ │ ├─┬ collection-visit@1.0.0
│ │ │ │ │ │ │ ├─┬ map-visit@1.0.0
│ │ │ │ │ │ │ │ └── object-visit@1.0.1 deduped
│ │ │ │ │ │ │ └─┬ object-visit@1.0.1
│ │ │ │ │ │ │ └── isobject@3.0.1 deduped
│ │ │ │ │ │ ├── component-emitter@1.3.0 deduped
│ │ │ │ │ │ ├── get-value@2.0.6
│ │ │ │ │ │ ├─┬ has-value@1.0.0
│ │ │ │ │ │ │ ├── get-value@2.0.6 deduped
│ │ │ │ │ │ │ ├─┬ has-values@1.0.0
│ │ │ │ │ │ │ │ ├── is-number@3.0.0 deduped
│ │ │ │ │ │ │ │ └─┬ kind-of@4.0.0
│ │ │ │ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ │ │ │ └── isobject@3.0.1 deduped
│ │ │ │ │ │ ├── isobject@3.0.1 deduped
│ │ │ │ │ │ ├─┬ set-value@2.0.1
│ │ │ │ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ │ │ │ └── is-extendable@0.1.1 deduped
│ │ │ │ │ │ │ ├── is-extendable@0.1.1 deduped
│ │ │ │ │ │ │ ├── is-plain-object@2.0.4 deduped
│ │ │ │ │ │ │ └── split-string@3.1.0 deduped
│ │ │ │ │ │ ├─┬ to-object-path@0.3.0
│ │ │ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ │ │ ├─┬ union-value@1.0.1
│ │ │ │ │ │ │ ├── arr-union@3.1.0 deduped
│ │ │ │ │ │ │ ├── get-value@2.0.6 deduped
│ │ │ │ │ │ │ ├── is-extendable@0.1.1 deduped
│ │ │ │ │ │ │ └── set-value@2.0.1 deduped
│ │ │ │ │ │ └─┬ unset-value@1.0.0
│ │ │ │ │ │ ├─┬ has-value@0.3.1
│ │ │ │ │ │ │ ├── get-value@2.0.6 deduped
│ │ │ │ │ │ │ ├── has-values@0.1.4
│ │ │ │ │ │ │ └─┬ isobject@2.1.0
│ │ │ │ │ │ │ └── isarray@1.0.0
│ │ │ │ │ │ └── isobject@3.0.1 deduped
│ │ │ │ │ ├─┬ class-utils@0.3.6
│ │ │ │ │ │ ├── arr-union@3.1.0
│ │ │ │ │ │ ├─┬ define-property@0.2.5
│ │ │ │ │ │ │ └── is-descriptor@0.1.6 deduped
│ │ │ │ │ │ ├── isobject@3.0.1 deduped
│ │ │ │ │ │ └─┬ static-extend@0.1.2
│ │ │ │ │ │ ├─┬ define-property@0.2.5
│ │ │ │ │ │ │ └── is-descriptor@0.1.6 deduped
│ │ │ │ │ │ └─┬ object-copy@0.1.0
│ │ │ │ │ │ ├── copy-descriptor@0.1.1
│ │ │ │ │ │ ├─┬ define-property@0.2.5
│ │ │ │ │ │ │ └── is-descriptor@0.1.6 deduped
│ │ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ │ ├── component-emitter@1.3.0
│ │ │ │ │ ├─┬ define-property@1.0.0
│ │ │ │ │ │ └─┬ is-descriptor@1.0.2
│ │ │ │ │ │ ├─┬ is-accessor-descriptor@1.0.0
│ │ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ │ ├─┬ is-data-descriptor@1.0.0
│ │ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ │ └── kind-of@6.0.2 deduped
│ │ │ │ │ ├── isobject@3.0.1 deduped
│ │ │ │ │ ├─┬ mixin-deep@1.3.2
│ │ │ │ │ │ ├── for-in@1.0.2
│ │ │ │ │ │ └─┬ is-extendable@1.0.1
│ │ │ │ │ │ └── is-plain-object@2.0.4 deduped
│ │ │ │ │ └── pascalcase@0.1.1
│ │ │ │ ├─┬ debug@2.6.9
│ │ │ │ │ └── ms@2.0.0
│ │ │ │ ├─┬ define-property@0.2.5
│ │ │ │ │ └─┬ is-descriptor@0.1.6
│ │ │ │ │ ├─┬ is-accessor-descriptor@0.1.6
│ │ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ │ ├─┬ is-data-descriptor@0.1.4
│ │ │ │ │ │ └─┬ kind-of@3.2.2
│ │ │ │ │ │ └── is-buffer@1.1.6 deduped
│ │ │ │ │ └── kind-of@5.1.0
│ │ │ │ ├─┬ extend-shallow@2.0.1
│ │ │ │ │ └── is-extendable@0.1.1 deduped
│ │ │ │ ├── map-cache@0.2.2 deduped
│ │ │ │ ├── source-map@0.5.7
│ │ │ │ ├─┬ source-map-resolve@0.5.2
│ │ │ │ │ ├── atob@2.1.2
│ │ │ │ │ ├── decode-uri-component@0.2.0
│ │ │ │ │ ├── resolve-url@0.2.1
│ │ │ │ │ ├── source-map-url@0.4.0
│ │ │ │ │ └── urix@0.1.0
│ │ │ │ └── use@3.1.1
│ │ │ └─┬ to-regex@3.0.2
│ │ │ ├── define-property@2.0.2 deduped
│ │ │ ├── extend-shallow@3.0.2 deduped
│ │ │ ├── regex-not@1.0.2 deduped
│ │ │ └── safe-regex@1.1.0 deduped
│ │ ├─┬ glob@7.1.5
│ │ │ ├── fs.realpath@1.0.0
│ │ │ ├─┬ inflight@1.0.6
│ │ │ │ ├── once@1.4.0 deduped
│ │ │ │ └── wrappy@1.0.2
│ │ │ ├── inherits@2.0.4
│ │ │ ├─┬ minimatch@3.0.4
│ │ │ │ └─┬ brace-expansion@1.1.11
│ │ │ │ ├── balanced-match@1.0.0
│ │ │ │ └── concat-map@0.0.1
│ │ │ ├─┬ once@1.4.0
│ │ │ │ └── wrappy@1.0.2 deduped
│ │ │ └── path-is-absolute@1.0.1
│ │ ├── ignore@4.0.6
│ │ ├── pify@4.0.1 deduped
│ │ └── slash@2.0.0
│ └── nested-error-stacks@2.1.0
└─┬ meow@5.0.0
├─┬ camelcase-keys@4.2.0
│ ├── camelcase@4.1.0
│ ├── map-obj@2.0.0
│ └── quick-lru@1.1.0
├─┬ decamelize-keys@1.1.0
│ ├── decamelize@1.2.0
│ └── map-obj@1.0.1
├─┬ loud-rejection@1.6.0
│ ├─┬ currently-unhandled@0.4.1
│ │ └── array-find-index@1.0.2
│ └── signal-exit@3.0.2
├─┬ minimist-options@3.0.2
│ ├── arrify@1.0.1 deduped
│ └── is-plain-obj@1.1.0
├─┬ normalize-package-data@2.5.0
│ ├── hosted-git-info@2.8.5
│ ├─┬ resolve@1.12.0
│ │ └── path-parse@1.0.6
│ ├── semver@5.7.1
│ └─┬ validate-npm-package-license@3.0.4
│ ├─┬ spdx-correct@3.1.0
│ │ ├── spdx-expression-parse@3.0.0 deduped
│ │ └── spdx-license-ids@3.0.5
│ └─┬ spdx-expression-parse@3.0.0
│ ├── spdx-exceptions@2.2.0
│ └── spdx-license-ids@3.0.5 deduped
├─┬ read-pkg-up@3.0.0
│ ├─┬ find-up@2.1.0
│ │ └─┬ locate-path@2.0.0
│ │ ├─┬ p-locate@2.0.0
│ │ │ └─┬ p-limit@1.3.0
│ │ │ └── p-try@1.0.0
│ │ └── path-exists@3.0.0
│ └─┬ read-pkg@3.0.0
│ ├─┬ load-json-file@4.0.0
│ │ ├── graceful-fs@4.2.3 deduped
│ │ ├─┬ parse-json@4.0.0
│ │ │ ├─┬ error-ex@1.3.2
│ │ │ │ └── is-arrayish@0.2.1
│ │ │ └── json-parse-better-errors@1.0.2
│ │ ├── pify@3.0.0
│ │ └── strip-bom@3.0.0
│ ├── normalize-package-data@2.5.0 deduped
│ └── path-type@3.0.0 deduped
├─┬ redent@2.0.0
│ ├── indent-string@3.2.0
│ └── strip-indent@2.0.0
├── trim-newlines@2.0.0
└─┬ yargs-parser@10.1.0
└── camelcase@4.1.0 dedupedYea, what the actually Effing F!?
197 dependencies, 1170 files, 47000 lines of JavaScript to copy files.
I ended up writing my own. There's the entire program
const fs = require('fs');
const src = process.argv[2];
const dst = process.argv[3];
fs.copyFileSync(src, dst);And I added it to my build like this
"scripts": {
"build": "make -f makefile && node copy.js a.out dist/MyApp",
...So, my first reaction was, yea, something is massively over engineered. Or maybe that's under engineered if by under engineered it means "made without thinking".
You might think so what, people have large hard drives, fast internet, lots of memory. Who cares about dependencies? Well, the more dependencies you have the more you get messages like this
found 35 vulnerabilities (1 low, 2 moderate, 31 high, 1 critical) in 1668 scanned packages
You get more and more and more maintenance with more dependencies.
Not only that, you get dependent, not just on the software but on the people maintaining that software. Above, 197 dependencies also means trusting none of them are doing anything bad. As far as we know one of those dependencies could easily have a time bomb waiting until some day in the future to pown your machine or server.
On the other hand my copy copies a single file. cpy-cli copies similar to cp.
It can copy multiple files and whole trees.
I started wondering what it would take to add the minimal features to reproduce
a functional cp clone. Note: not a full clone, a functional clone I'm sure
cp has a million features but in my entire 40yr career I've only used about 2
of those features. (1) copying using wildcard as in cp *.txt dst which honestly
is handled by the shell, not cp. (2) copying recursively cp -R src dst.
The first thing I did was look at a command line argument library. I've used
one called optionator in the past and it's fine. I check and it has several
dependencies. 2 that stick out are:
a wordwrap library.
This is used to make your command's help fit the size of the terminal you're in. Definitely a useful feature. I have terminals of all difference sizes. I default to having 4 open.
a levenshtein distance library.
This is used so that if you specify a switch that doesn't exist it can try to suggest the correct one. For example might type:
my-copy-clone --src=abc.txt -destinatoin=def.txt
and it would says something like
no such switch: 'destinatoin' did you mean 'destination'?`.
Yea, that's kind of useful too.
Okay so my 4 line copy.js just got 3500 lines of libraries added. Or maybe I should look into another library that uses less deps while getting "woke" about dependencies.
Meh, I decide to parse my own arguments rather that take 3500 lines of code and 7 dependencies. Here's the code
#!/usr/bin/env node
'use strict';
const fs = require('fs');
const ldcp = require('../src/ldcp');
const args = process.argv.slice(2);
const options = {
recurse: false,
dryRun: false,
verbose: false,
};
while (args.length && args[0].startsWith('-')) {
const opt = args.shift();
switch (opt) {
case '-v':
case '--verbose':
options.verbose = true;
break;
case '--dry-run':
options.dryRun = true;
options.verbose = true;
break;
case '-R':
options.recurse = true;
break;
default:
console.error('illegal option:', opt);
printUsage();
}
}
function printUsage() {
console.log('usage: ldcp [-R] src_file dst_file\n ldcp [-R] src_file ... dst_dir');
process.exit(1);
}
let dst = args.pop();
if (args.length < 1) {
printUsage();
}Now that the args are parsed we need a function to copy the files
const path = require('path');
const fs = require('fs');
const defaultAPI = {
copyFileSync(...args) { return fs.copyFileSync(...args) },
mkdirSync(...args) { return fs.mkdirSync(...args); },
statSync(...args) { return fs.statSync(...args); },
readdirSync(...args) { return fs.readdirSync(...args); },
log() {},
};
function ldcp(_srcs, dst, options, api = defaultAPI) {
const {recurse} = options;
// check if dst is or needs to be a directory
const dstStat = safeStat(dst);
let isDstDirectory = false;
let needMakeDir = false;
if (dstStat) {
isDstDirectory = dstStat.isDirectory();
} else {
isDstDirectory = recurse;
needMakeDir = recurse;
}
if (!recurse && _srcs.length > 1 && !isDstDirectory) {
throw new Error('can not copy multiple files to same dst file');
}
const srcs = [];
// handle the case where src ends with / like cp
for (const src of _srcs) {
if (recurse) {
const srcStat = safeStat(src);
if ((needMakeDir && srcStat && srcStat.isDirectory()) ||
(src.endsWith('/') || src.endsWith('\\'))) {
srcs.push(...api.readdirSync(src).map(f => path.join(src, f)));
continue;
}
}
srcs.push(src);
}
const srcDsts = [{srcs, dst, isDstDirectory, needMakeDir}];
while (srcDsts.length) {
const {srcs, dst, isDstDirectory, needMakeDir} = srcDsts.shift();
if (needMakeDir) {
api.log('mkdir', dst);
api.mkdirSync(dst);
}
for (const src of srcs) {
const dstFilename = isDstDirectory ? path.join(dst, path.basename(src)) : dst;
if (recurse) {
const srcStat = api.statSync(src);
if (srcStat.isDirectory()) {
srcDsts.push({
srcs: api.readdirSync(src).map(f => path.join(src, f)),
dst: path.join(dst, path.basename(src)),
isDstDirectory: true,
needMakeDir: true,
});
continue;
}
}
api.log('copy', src, dstFilename);
api.copyFileSync(src, dstFilename);
}
}
function safeStat(filename) {
try {
return api.statSync(filename.replace(/(\\|\/)$/, ''));
} catch (e) {
//
}
}
}I made it so you pass an optional API of all the external functions it calls. That way you can pass in for example functions that do nothing if you want to test it. Or you can pass in graceful-fs if that's your jam but in the interest of NOT adding dependencies if you want that that's on you. Simple!
All that's left is using it after parsing the args
const log = options.verbose ? console.log.bind(console) : () => {};
const api = options.dryRun ? {
copyFileSync(src) { fs.statSync(src) },
mkdirSync() { },
statSync(...args) { return fs.statSync(...args); },
readdirSync(...args) { return fs.readdirSync(...args); },
log,
} : {
copyFileSync(...args) { return fs.copyFileSync(...args) },
mkdirSync(...args) { return fs.mkdirSync(...args); },
statSync(...args) { return fs.statSync(...args); },
readdirSync(...args) { return fs.readdirSync(...args); },
log,
};
ldcp(args, dst, options, api);Total lines: 176 and 0 dependencies.
It's here if you want it.