It seems that when I override a python3 package in nixpkgs.config.packageOverrides or nixpkgs.overlays, a python application in environment.systemPackages is not using those overrides. How can I get that overridden python3 package to be used in a python3 application?
Example darwin-configuration.nix that uses nixpkgs.overlays:
{ config, pkgs, lib, ... }:
{
environment.systemPackages =
[
pkgs.myawscli2
];
nixpkgs.overlays = let overlayRemovePyopenssl = self: super:
let removePyopenssl = pythonpkgs:
lib.filter
(pythonpkg: !(pythonpkg != null && pythonpkg ? pname && pythonpkg.pname == "pyopenssl"))
pythonpkgs;
in {
python3 = super.python3.override {
packageOverrides = python-self: python-super: rec {
# Delete pyopenssl; workaround for broken package on darwin-aarch64
# “Package ‘python3.10-pyopenssl-22.0.0’ in /nix/store/<hash>-nixpkgs/nixpkgs/pkgs/development/python-modules/pyopenssl/default.nix:73 is marked as broken, refusing to evaluate”
# https://github.com/NixOS/nixpkgs/issues/174457
urllib3 = python-super.urllib3.overridePythonAttrs (origattrs: rec {
propagatedBuildInputs = removePyopenssl origattrs.propagatedBuildInputs;
});
twisted = python-super.twisted.overridePythonAttrs (origattrs: {
checkInputs = removePyopenssl origattrs.checkInputs;
});
};
};
myawscli2 = (self.awscli2.override {
# override the python3 arg of awscli2
# https://github.com/NixOS/nixpkgs/blob/f72be3af76fb7dc45e2088d8cb9aba1e6767a930/pkgs/tools/admin/awscli2/default.nix#L2
python3 = self.python3;
});
}; in
[
overlayRemovePyopenssl
];
system.stateVersion = 4;
}
The overlay is applied properly in the individual python package urllib3:
nix-repl> lib = import <nixpkgs>.lib
nix-repl> :l <darwin>
nix-repl> lib.forEach pkgs.python3.pkgs.urllib3.propagatedBuildInputs (x: x.pname)
[ "brotli" "certifi" "cryptography" "idna" "python3" ]
However, the overlay was not applied to the python application that uses urllib3. Note that pyopenssl is in the dependencies of urllib3 when used by awscli2:
nix-repl> lib.forEach (lib.findFirst (x: x.pname == "urllib3") null pkgs.awscli2.propagatedBuildInputs).propagatedBuildInputs (x: x.pname)
[ "brotli" "certifi" "cryptography" "idna" "pyopenssl" "python3" ]
I also tried the same thing with nixpkgs.config.packageOverrides with the same effect:
{ config, pkgs, lib, ... }:
{
environment.systemPackages =
[
pkgs.myawscli2
];
nixpkgs.config.packageOverrides = super:
let removePyopenssl = pythonpkgs:
lib.filter
(pythonpkg: !(pythonpkg != null && lib.hasAttr "pname" pythonpkg && pythonpkg.pname == "pyopenssl"))
pythonpkgs;
in {
python3 = super.python3.override {
packageOverrides = python-self: python-super: rec {
# workaround for
# “Package ‘python3.10-pyopenssl-22.0.0’ in /nix/store/<hash>-nixpkgs/nixpkgs/pkgs/development/python-modules/pyopenssl/default.nix:73 is marked as broken, refusing to evaluate”
# https://github.com/NixOS/nixpkgs/issues/174457
urllib3 = python-super.urllib3.overridePythonAttrs (origattrs: rec {
propagatedBuildInputs = removePyopenssl origattrs.propagatedBuildInputs;
});
twisted = python-super.twisted.overridePythonAttrs (origattrs: {
checkInputs = removePyopenssl origattrs.checkInputs;
});
};
};
myawscli2 = (pkgs.awscli2.override {
python3 = pkgs.python3;
});
};
system.stateVersion = 4;
}
This seems to contradict the nixpkgs manual, which says: “pythonPackages.twisted is now globally overridden. All packages and also all NixOS services that reference twisted (such as services.buildbot-worker) now use the new definition”
I also asked this on Element. The problem is that awscli2/default.nix calls python3.override {packageOverrides = …;}:
py = python3.override {
packageOverrides = self: super: {
…
};
};
This unfortunately overwrites my own .override {packageOverrides = …;}. Unfortunately you can’t compose (python.override {packageOverrides = …;}).override {packageOverrides = …;}
I worked around this by checking out nixpkgs and editing the awscli2 nix to combine the python application’s packageOverrides with packageOverrides from the arguments:
py = python3.override (oldargs: {
packageOverrides = self: super: {
…
} // (if oldargs?packageOverrides then (oldargs.packageOverrides self super) else super);
});
I have a list of ~46 dependencies.
I produce a shell using among other things a call to:
server = pkgs.haskellPackages.callCabal2nix "server" ./server.cabal { };
within an override of pkgs.haskellPackages that goes into pkgs.myHaskellPackages.
The shell works fine. However, I have another nix for building a docker and this requires a build input that comes from:
ghc = pkgs.myHaskellPackages.ghcWithPackages (p: [ longlistofdependencies ]);
How do I specify the long list of dependencies without having to write them out manually, defeating the purpose of using callCabal2nix in the first place.
let
bootstrap = import <nixpkgs> { };
nixpkgs = builtins.fromJSON (builtins.readFile ./nixpkgs-unstable.json);
src = bootstrap.fetchFromGitHub {
owner = "NixOS";
repo = "nixpkgs";
inherit (nixpkgs) rev sha256;
};
config = {
allowBroken = true;
packageOverrides = pkgs: rec {
servantSrc = pkgs.fetchFromGitHub {
owner = "haskell-servant";
repo = "servant";
rev = "73c87bc2bc0685649f2337b06ab4fdc66c4ce1dd";
sha256 = "0sw4mrncmfna30cyxrvinc1krqhfxn5dcc1ggzqfy39s0yl9q98r";
#sha256 = "0000000000000000000000000000000000000000000000000000";
} + "/servant";
myHaskellPackages = pkgs.haskellPackages.override {
overrides = haskellPackagesNew: haskellPackagesOld: rec {
server =
haskellPackagesNew.callCabal2nix "server" ./server.cabal { };
servant =
haskellPackagesNew.callCabal2nix "servant" servantSrc {};
# several other overridden packages
};
};
};
};
pkgs = import src { inherit config; };
devShell = pkgs.myHaskellPackages.shellFor {
packages = p: [
p.server
];
buildInputs = with pkgs.haskellPackages; [
hlint
brittany
haskell-language-server
];
};
# Every time a dependency changes in the cabal file, this has to be edited as well.
ghc = pkgs.myHaskellPackages.ghcWithPackages (p: [
p.aeson p.aeson-qq p.base p.blaze-html p.bytestring p.cache p.containers
p.contravariant-extras p.criterion p.either p.hashable p.hasql
p.hasql-migration p.hasql-pool p.hasql-th p.hasql-transaction p.hspec
p.http-client p.http-conduit p.http-types p.immortal-queue p.lens p.linear
p.monad-logger p.mtl p.pqueue p.pretty-simple p.QuickCheck p.raw-strings-qq
p.servant-blaze p.servant-docs p.servant-server
p.string-interpolate p.text p.text-format p.time p.tuple p.unordered-containers
p.utf8-string p.vector p.vector-split p.wai p.wai-cors p.warp p.xlsx p.yaml
]);
in {
inherit devShell;
inherit ghc;
inherit pkgs;
}
The ghc output is used in docker.nix as a build input:
let
inherit (import ./pinned-nixpkgs.nix) ghc pkgs;
pricing-server =
pkgs.stdenv.mkDerivation {
name = "my-server";
pname = "my-server";
version = "1.1.0";
src = ./.;
buildPhase = ''
ghc -O2 --make -outputdir ./tmp2 Main.hs
'';
installPhase = ''
mkdir -p $out/bin
cp Main $out/bin
cp -r ./sql $out/bin
'';
buildInputs = [ ghc ];
};
in
dockerTools.buildImage {
# irrelevant to the question.
}
A possible solution is pkgs.myHaskellPackages.server.getBuildInputs.haskellBuildInputs, or pkgs.myHaskellPackages.server.getCabalDeps.libraryHaskellDepends.
You can explore these attributes, or any expression, with nix repl. You may have to expose some values from your let bindings though. In this case I just browsed through haskellPackages.warp in nix repl <nixpkgs>.
I also noticed you use rec in an overlay. This will work for you until it doesn't. I'd recommend to remove rec to avoid accessing attributes in a third way and use the more standard haskellPackagesNew.servant instead.
I'm using Jooq and gradle-jooq-plugin for code generation. It works fine, but I'm having a problem getting the generated code to update when a table is added or a column is dropped. I was able to force an update by changing the "packageName" config parameter and build a new package. And by going back to the original name the code was updated as expected.
What would be the correct way to re-generate code after schema change with my setup?
jooq {
version = '3.13.1'
edition = 'OSS'
generateSchemaSourceOnCompilation = true
sample(sourceSets.main) {
jdbc {
driver = 'org.postgresql.Driver'
url = 'jdbc:postgresql://0.0.0.0:5432/victor'
user = 'postgres'
password = 'docker'
properties {
property {
key = 'ssl'
value = 'false'
}
}
}
generator {
name = 'org.jooq.codegen.DefaultGenerator'
strategy {
name = 'org.jooq.codegen.DefaultGeneratorStrategy'
}
database {
name = 'org.jooq.meta.postgres.PostgresDatabase'
inputSchema = 'public'
forcedTypes {
forcedType {
name = 'varchar'
expression = '.*'
types = 'INET'
}
}
}
generate {
relations = true
deprecated = false
records = true
immutablePojos = true
fluentSetters = true
}
target {
packageName = 'net.bravo.victor.model'
directory = 'src/'
}
}
}
I'm using https://github.com/etiennestuder/gradle-jooq-plugin
plugins {
id 'nu.studer.jooq' version '4.1'
}
I am not sure whether it is correct way but for me works this:
generateNavigoJooqSchemaSource {
dependsOn cleanGenerateNavigoJooqSchemaSource
}
task buildJooq(dependsOn: generateNavigoJooqSchemaSource)
So I have created task name (buildJooq) I can remember that depends on generate task (generateNavigoJooqSchemaSource) and that depends on clean (cleanGenerateNavigoJooqSchemaSource) task.
Previously I have used this code which works too:
tasks.named("generateNavigoJooqSchemaSource").configure {
outputs.upToDateWhen { false }
}
It also forces run every time.
I would like to understand best practice for accessing variables that are defined in an external module that would then be required in several other files.
Say we have the two following files, I would like to access the value of the h variable in consume.js
//central.js
module.exports = {
common: function() {
var h = "hello";
var b = "enabled"
}
};
and
//consume.js
var g = require('./central');
//get the value of variable h from central.js
Taking this a step further, if I have the following consume.js, fetch.js and get.js files that all imported central.js, and required a common set of variables from central.js, how does one go about defining these common variables inside central.js so that the dependent files can consume them?
Thanks!
Export:
// central.js
module.exports = {
h: "hello",
b: "enabled"
}
Import:
// consume.js
const { h, b } = require('./central')
Or alternatively:
// get.js
const central = require('./central')
// central.h
// central.b
Hope this helps!
My yeoman generator copies files from template to destination path:
this.fs.copyTpl(
this.templatePath(),
this.destinationPath(), {
appName: this.props.appName
});
During project generation, I need to assign value of this.props.appName to some of filenames.
Unfortunately I can't do this that way like I could do inside this files:
<%=appName%>-project.sln
All files that need to be renamed have appTemplate in their names, so what I need to do is simply replace appTemplate with value of this.props.appName.
Can I somehow configure copyTpl to rename some of files while copying them to another destination?
OK, I found a solution. According to yeoman docs:
Any generator author can register a transformStream to modify the file path and/or the content.
Using this method:
this.registerTransformStream();
What that means is I can pipe all generated files through some script:
var rename = require("gulp-rename");
//other dependecies...
module.exports = yeoman.Base.extend({
//some other things generator do...
writing: function() {
var THAT = this;
this.registerTransformStream(rename(function(path) {
path.basename = path.basename.replace(/(666replacethat666)/g, THAT.props.appName);
path.dirname = path.dirname.replace(/(666replacethat666)/g, THAT.props.appName);
}));
this.fs.copyTpl(
this.templatePath(),
this.destinationPath(), {
appName: this.props.appName
});
}
});
This script will pipe all files through gulp-rename, changing 666replacethat666 to something more intelligent.
If you cannot use registerTransformStream because you are using the composeWith() feature in Yeoman (which disconnects transform stream registrations), you can use the processDestinationPath, which works when you select multiple files (not when you specify a specific file in the first argument, for some reason).
this.fs.copyTpl(
this.templatePath("**/{.*,*}"),
this.destinationPath(),
{ /* usually your prompt answers are here */ },
{},
{
processDestinationPath: (filePath: string) =>
filePath.replace(/somedir\/a-file.js/g, 'newdir/better-filename.js'),
},
);
Source to documentation options: https://yeoman.github.io/generator/actions_fs.html#.copyTemplate
Which is based on https://github.com/SBoudrias/mem-fs-editor#copyfrom-to-options-context-templateoptions-
registerTransformStream with gulp-rename is still an issue. However, I get it working with glob.
const glob = require('glob');
writing() {
const files = glob.sync('**', { dot: true, nodir: true, cwd: this.templatePath() })
for (let i in files) {
this.fs.copyTpl(
this.templatePath(files[i]),
this.destinationPath( this.props.destinationFolderPath + '\\' + files[i].replace(/__fileName__/g,this.props.fileName)),
this.props
)
}
}
After copy, iterate over the paths of the output dir and regex replace all occurrences.
const getReplacement = (base, pathRel, match, replace) => {
let pathRelNew = pathRel.replace(match, replace);
let oldPathAbs = path.join(base, pathRel);
let newPathAbs = path.join(base, pathRelNew);
if (oldPathAbs != newPathAbs) {
return {
oldPath: oldPathAbs,
newPath: newPathAbs
}
}
}
const getReplacementsRecursive = (base, match, replace, replacements = []) => {
let pathsRel = fs.readdirSync(base);
pathsRel.forEach(pathRel => {
if (fs.statSync(path.join(base, pathRel)).isDirectory()) {
replacements = getReplacementsRecursive(path.join(base, pathRel), match, replace, replacements);
var replacement = getReplacement(base, pathRel, match, replace)
if (replacement) replacements.push(replacement);
} else {
var replacement = getReplacement(base, pathRel, match, replace)
if (replacement) replacements.push(replacement);
}
});
return replacements;
};
function replaceMatches(dir, match, replace) {
var replacements = getReplacementsRecursive(dir, match, replace);
replacements.forEach(function(replacement) {
fs.renameSync(replacement.oldPath, replacement.newPath);
});
}
module.exports = class extends Generator {
// ...
writing() {
// don't forget to set the output directory
let OUTPUT_DIR = "./out";
// this.fs.copyTpl(...);
// setTimeout is used to give some time for the copyTpl to finish
setTimeout(
() => {
var match = new RegExp( "666replacethat666", 'g' );
replaceMatches(OUTPUT_DIR, match, this.props.appName);
}, 1000);
}
}