GNU bug report logs -
#52078
[PATCH 1/2] gnu: julia: Disable broken tests.
Previous Next
Reported by: zimoun <zimon.toutoune <at> gmail.com>
Date: Wed, 24 Nov 2021 14:20:02 UTC
Severity: normal
Tags: patch
Done: Maxim Cournoyer <maxim.cournoyer <at> gmail.com>
Bug is archived. No further changes may be made.
To add a comment to this bug, you must first unarchive it, by sending
a message to control AT debbugs.gnu.org, with unarchive 52078 in the body.
You can then email your comments to 52078 AT debbugs.gnu.org in the normal way.
Toggle the display of automated, internal messages from the tracker.
Report forwarded
to
guix-patches <at> gnu.org
:
bug#52078
; Package
guix-patches
.
(Wed, 24 Nov 2021 14:20:02 GMT)
Full text and
rfc822 format available.
Acknowledgement sent
to
zimoun <zimon.toutoune <at> gmail.com>
:
New bug report received and forwarded. Copy sent to
guix-patches <at> gnu.org
.
(Wed, 24 Nov 2021 14:20:02 GMT)
Full text and
rfc822 format available.
Message #5 received at submit <at> debbugs.gnu.org (full text, mbox):
* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
gnu/packages/julia.scm | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..0d4dd2b7f2 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -382,7 +382,18 @@ (define-public julia
;(setenv "LC_ALL" "en_US.utf8")
(substitute* "test/cmdlineargs.jl"
(("test v\\[3") "test_broken v[3")
- (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+ (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+ ;; These tests randomly fails because they depend on CPU.
+ (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+ ;; Fixed in v1.6.4
+ ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+ (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+ "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+ (substitute* "test/math.jl"
+ ;; @test_broken cannot be used because if the test randomly
+ ;; passes, then it also raises an error.
+ (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+ " "))))
(add-before 'install 'symlink-libraries
(lambda* (#:key inputs outputs #:allow-other-keys)
(let ((link
base-commit: 6605d9ba356f7975e75c0664d9ef02bb7bc9cc1e
--
2.32.0
Information forwarded
to
guix-patches <at> gnu.org
:
bug#52078
; Package
guix-patches
.
(Wed, 24 Nov 2021 14:21:01 GMT)
Full text and
rfc822 format available.
Message #8 received at 52078 <at> debbugs.gnu.org (full text, mbox):
* gnu/packages/julia.scm (julia)[origin]: Patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
---
gnu/packages/julia.scm | 3 ++-
.../patches/julia-Allow-parallel-build.patch | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
create mode 100644 gnu/packages/patches/julia-Allow-parallel-build.patch
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 0d4dd2b7f2..2394cb3d5a 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
(base32
"1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
(patches
- (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+ (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+ "julia-Allow-parallel-build.patch"))))
(build-system gnu-build-system)
(arguments
`(#:test-target "test"
diff --git a/gnu/packages/patches/julia-Allow-parallel-build.patch b/gnu/packages/patches/julia-Allow-parallel-build.patch
new file mode 100644
index 0000000000..1eb6a4f1f1
--- /dev/null
+++ b/gnu/packages/patches/julia-Allow-parallel-build.patch
@@ -0,0 +1,18 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..d8d6d15b8c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -83,8 +83,11 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+ n = 1
+- if net_on
+- n = min(Sys.CPU_THREADS, length(tests))
++ # Allow parallel tests with isolated environment
++ # https://github.com/JuliaLang/julia/issues/43205
++ if net_on || haskey(ENV, "JULIA_CPU_THREADS")
++ x = haskey(ENV, "JULIA_CPU_THREADS") ? parse(Int, ENV["JULIA_CPU_THREADS"]) : Sys.CPU_THREADS
++ n = min(x, length(tests))
+ n > 1 && addprocs_with_testenv(n)
+ LinearAlgebra.BLAS.set_num_threads(1)
+ end
--
2.32.0
Information forwarded
to
guix-patches <at> gnu.org
:
bug#52078
; Package
guix-patches
.
(Thu, 25 Nov 2021 01:25:02 GMT)
Full text and
rfc822 format available.
Message #11 received at 52078 <at> debbugs.gnu.org (full text, mbox):
* gnu/packages/julia.scm (julia)[origin]<patches>: New patch.
* gnu/packages/patches/julia-Allow-parallel-build.patch: New file.
* gnu/local.mk (dist_patch_DATA): Register it.
---
gnu/local.mk | 2 ++
gnu/packages/julia.scm | 3 +-
.../patches/julia-allow-parallel-build.patch | 30 +++++++++++++++++++
3 files changed, 34 insertions(+), 1 deletion(-)
create mode 100644 gnu/packages/patches/julia-allow-parallel-build.patch
diff --git a/gnu/local.mk b/gnu/local.mk
index ea8a203ac9..706725965b 100644
--- a/gnu/local.mk
+++ b/gnu/local.mk
@@ -46,6 +46,7 @@
# Copyright © 2021 Sharlatan Hellseher <sharlatanus <at> gmail.com>
# Copyright © 2021 Dmitry Polyakov <polyakov <at> liltechdude.xyz>
# Copyright © 2021 Andrew Tropin <andrew <at> trop.in>
+# Copyright © 2021 Simon Tournier <zimon.toutoune <at> gmail.com>
#
# This file is part of GNU Guix.
#
@@ -1296,6 +1297,7 @@ dist_patch_DATA = \
%D%/packages/patches/jfsutils-include-systypes.patch \
%D%/packages/patches/julia-SOURCE_DATE_EPOCH-mtime.patch \
%D%/packages/patches/julia-tracker-16-compat.patch \
+ %D%/packages/patches/julia-allow-parallel-build.patch \
%D%/packages/patches/kdbusaddons-kinit-file-name.patch \
%D%/packages/patches/libffi-3.3-powerpc-fixes.patch \
%D%/packages/patches/libffi-float128-powerpc64le.patch \
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index 649ffb8ee4..cf76cfa23c 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -145,7 +145,8 @@ (define-public julia
(base32
"1515x8fs25l3f9csbmd1v4nm041zvjnvigy6s5iidy4yrkwdx4r5"))
(patches
- (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"))))
+ (search-patches "julia-SOURCE_DATE_EPOCH-mtime.patch"
+ "julia-allow-parallel-build.patch"))))
(build-system gnu-build-system)
(arguments
`(#:test-target "test"
diff --git a/gnu/packages/patches/julia-allow-parallel-build.patch b/gnu/packages/patches/julia-allow-parallel-build.patch
new file mode 100644
index 0000000000..5a382e7c89
--- /dev/null
+++ b/gnu/packages/patches/julia-allow-parallel-build.patch
@@ -0,0 +1,30 @@
+diff --git a/test/runtests.jl b/test/runtests.jl
+index 2f9cd058bb..150395e78c 100644
+--- a/test/runtests.jl
++++ b/test/runtests.jl
+@@ -4,7 +4,7 @@ using Test
+ using Distributed
+ using Dates
+ import REPL
+-using Printf: @sprintf
++using Printf: @sprintf, @printf
+ using Base: Experimental
+
+ include("choosetests.jl")
+@@ -83,11 +83,15 @@ prepend!(tests, linalg_tests)
+ import LinearAlgebra
+ cd(@__DIR__) do
+ n = 1
+- if net_on
++ # Allow parallel tests with isolated environment
++ # https://github.com/JuliaLang/julia/issues/43205
++ # https://github.com/JuliaLang/julia/pull/43211
++ if net_on || haskey(ENV, "JULIA_CPU_THREADS")
+ n = min(Sys.CPU_THREADS, length(tests))
+ n > 1 && addprocs_with_testenv(n)
+ LinearAlgebra.BLAS.set_num_threads(1)
+ end
++ @printf("Number of threads: %i\n", n)
+ skipped = 0
+
+ @everywhere include("testdefs.jl")
--
2.32.0
Information forwarded
to
guix-patches <at> gnu.org
:
bug#52078
; Package
guix-patches
.
(Thu, 25 Nov 2021 01:25:02 GMT)
Full text and
rfc822 format available.
Message #14 received at 52078 <at> debbugs.gnu.org (full text, mbox):
* gnu/packages/julia.scm (julia)[arguments]: Disable broken tests.
---
gnu/packages/julia.scm | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/gnu/packages/julia.scm b/gnu/packages/julia.scm
index aff7e1a66f..649ffb8ee4 100644
--- a/gnu/packages/julia.scm
+++ b/gnu/packages/julia.scm
@@ -358,8 +358,8 @@ (define-public julia
;; Dates/io tests fail on master when networking is unavailable
;; https://github.com/JuliaLang/julia/issues/34655
(substitute* "stdlib/Dates/test/io.jl"
- (("\"Dates.Date") "\"Date")
- (("\"Dates.Time") "\"Time"))
+ (("using Dates") "import Dates
+using Dates: @dateformat_str, Date, DateTime, DateFormat, Time"))
;; Upstream bug I found when packaging
;; https://github.com/JuliaLang/julia/issues/35785
(substitute* "test/file.jl"
@@ -382,7 +382,18 @@ (define-public julia
;(setenv "LC_ALL" "en_US.utf8")
(substitute* "test/cmdlineargs.jl"
(("test v\\[3") "test_broken v[3")
- (("test isempty\\(v\\[3") "test_broken isempty(v[3"))))
+ (("test isempty\\(v\\[3") "test_broken isempty(v[3"))
+ ;; These tests randomly fails because they depend on CPU.
+ (substitute* "stdlib/LinearAlgebra/test/matmul.jl"
+ ;; Fixed in v1.6.4
+ ;; https://github.com/JuliaLang/julia/blob/v1.6.4/stdlib/LinearAlgebra/test/matmul.jl#L155
+ (("@test mul\\!\\(C, vf, transpose\\(vf\\), 2, 3\\) == 2vf\\*vf' \\.\\+ 3C0")
+ "@test mul!(C, vf, transpose(vf), 2, 3) ≈ 2vf*vf' .+ 3C0"))
+ (substitute* "test/math.jl"
+ ;; @test_broken cannot be used because if the test randomly
+ ;; passes, then it also raises an error.
+ (("@test isinf\\(log1p\\(-one\\(T\\)\\)\\)")
+ " "))))
(add-before 'install 'symlink-libraries
(lambda* (#:key inputs outputs #:allow-other-keys)
(let ((link
base-commit: c70eadeaed9367e45be3797a17d3a34e5f8f67cd
--
2.32.0
Information forwarded
to
guix-patches <at> gnu.org
:
bug#52078
; Package
guix-patches
.
(Thu, 25 Nov 2021 01:25:03 GMT)
Full text and
rfc822 format available.
Message #17 received at 52078 <at> debbugs.gnu.org (full text, mbox):
The tests are defined by the script 'test/runtests.jl' from packages and the
parallelism depends on the implementation of this script. Therefore, 'julia'
is launched using local worker processes accordingly with 'parallel?'.
* guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
options '--procs'.
* guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
---
guix/build-system/julia.scm | 3 +++
guix/build/julia-build-system.scm | 9 ++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
index 2998663df1..6261f8a55a 100644
--- a/guix/build-system/julia.scm
+++ b/guix/build-system/julia.scm
@@ -2,6 +2,7 @@
;;; Copyright © 2019 Nicolò Balzarotti <nicolo <at> nixo.xyz>
;;; Copyright © 2021 Ludovic Courtès <ludo <at> gnu.org>
;;; Copyright © 2021 Jean-Baptiste Volatier <jbv <at> pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune <at> gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -79,6 +80,7 @@ (define private-keywords
(define* (julia-build name inputs
#:key source
(tests? #t)
+ (parallel-tests? #t)
(phases '%standard-phases)
(outputs '("out"))
(search-paths '())
@@ -98,6 +100,7 @@ (define builder
#:source #+source
#:system #$system
#:tests? #$tests?
+ #:parallel-tests? #$parallel-tests?
#:phases #$phases
#:outputs #$(outputs->gexp outputs)
#:search-paths '#$(sexp->gexp
diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
index 41c69665c6..f0dc419c17 100644
--- a/guix/build/julia-build-system.scm
+++ b/guix/build/julia-build-system.scm
@@ -1,6 +1,7 @@
;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo <at> nixo.xyz>
;;; Copyright © 2021 Jean-Baptiste Volatier <jbv <at> pm.me>
+;;; Copyright © 2021 Simon Tournier <zimon.toutoune <at> gmail.com>
;;;
;;; This file is part of GNU Guix.
;;;
@@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
#t)
(define* (check #:key tests? source inputs outputs julia-package-name
+ parallel-tests?
#:allow-other-keys)
(when tests?
(let* ((out (assoc-ref outputs "out"))
(package (or julia-package-name (project.toml->name "Project.toml")))
- (builddir (string-append out "/share/julia/")))
+ (builddir (string-append out "/share/julia/"))
+ (jobs (if parallel-tests?
+ (number->string (parallel-job-count))
+ "1")))
;; With a patch, SOURCE_DATE_EPOCH is honored
(setenv "SOURCE_DATE_EPOCH" "1")
(setenv "JULIA_DEPOT_PATH" builddir)
@@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
(string-append builddir "loadpath/" ":"
(or (getenv "JULIA_LOAD_PATH")
"")))
+ (setenv "JULIA_CPU_THREADS" jobs)
(setenv "HOME" "/tmp")
(invoke "julia" "--depwarn=yes"
+ (string-append "--procs=" jobs)
(string-append builddir "loadpath/"
package "/test/runtests.jl"))))
#t)
--
2.32.0
Reply sent
to
Maxim Cournoyer <maxim.cournoyer <at> gmail.com>
:
You have taken responsibility.
(Thu, 25 Nov 2021 06:44:02 GMT)
Full text and
rfc822 format available.
Notification sent
to
zimoun <zimon.toutoune <at> gmail.com>
:
bug acknowledged by developer.
(Thu, 25 Nov 2021 06:44:02 GMT)
Full text and
rfc822 format available.
Message #22 received at 52078-done <at> debbugs.gnu.org (full text, mbox):
Hello Simon,
zimoun <zimon.toutoune <at> gmail.com> writes:
> The tests are defined by the script 'test/runtests.jl' from packages and the
> parallelism depends on the implementation of this script. Therefore, 'julia'
> is launched using local worker processes accordingly with 'parallel?'.
>
> * guix/build/julia-build-system.scm (check): Set JULIA_CPU_THREADS, add
> options '--procs'.
> * guix/build-system/julia.scm (julia-build): Add 'parallel-tests?'.
> ---
> guix/build-system/julia.scm | 3 +++
> guix/build/julia-build-system.scm | 9 ++++++++-
> 2 files changed, 11 insertions(+), 1 deletion(-)
>
> diff --git a/guix/build-system/julia.scm b/guix/build-system/julia.scm
> index 2998663df1..6261f8a55a 100644
> --- a/guix/build-system/julia.scm
> +++ b/guix/build-system/julia.scm
> @@ -2,6 +2,7 @@
> ;;; Copyright © 2019 Nicolò Balzarotti <nicolo <at> nixo.xyz>
> ;;; Copyright © 2021 Ludovic Courtès <ludo <at> gnu.org>
> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv <at> pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune <at> gmail.com>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> @@ -79,6 +80,7 @@ (define private-keywords
> (define* (julia-build name inputs
> #:key source
> (tests? #t)
> + (parallel-tests? #t)
> (phases '%standard-phases)
> (outputs '("out"))
> (search-paths '())
> @@ -98,6 +100,7 @@ (define builder
> #:source #+source
> #:system #$system
> #:tests? #$tests?
> + #:parallel-tests? #$parallel-tests?
> #:phases #$phases
> #:outputs #$(outputs->gexp outputs)
> #:search-paths '#$(sexp->gexp
> diff --git a/guix/build/julia-build-system.scm b/guix/build/julia-build-system.scm
> index 41c69665c6..f0dc419c17 100644
> --- a/guix/build/julia-build-system.scm
> +++ b/guix/build/julia-build-system.scm
> @@ -1,6 +1,7 @@
> ;;; GNU Guix --- Functional package management for GNU
> ;;; Copyright © 2019, 2020 Nicolò Balzarotti <nicolo <at> nixo.xyz>
> ;;; Copyright © 2021 Jean-Baptiste Volatier <jbv <at> pm.me>
> +;;; Copyright © 2021 Simon Tournier <zimon.toutoune <at> gmail.com>
> ;;;
> ;;; This file is part of GNU Guix.
> ;;;
> @@ -103,11 +104,15 @@ (define* (precompile #:key source inputs outputs julia-package-name
> #t)
>
> (define* (check #:key tests? source inputs outputs julia-package-name
> + parallel-tests?
> #:allow-other-keys)
> (when tests?
> (let* ((out (assoc-ref outputs "out"))
> (package (or julia-package-name (project.toml->name "Project.toml")))
> - (builddir (string-append out "/share/julia/")))
> + (builddir (string-append out "/share/julia/"))
> + (jobs (if parallel-tests?
> + (number->string (parallel-job-count))
> + "1")))
> ;; With a patch, SOURCE_DATE_EPOCH is honored
> (setenv "SOURCE_DATE_EPOCH" "1")
> (setenv "JULIA_DEPOT_PATH" builddir)
> @@ -115,8 +120,10 @@ (define* (check #:key tests? source inputs outputs julia-package-name
> (string-append builddir "loadpath/" ":"
> (or (getenv "JULIA_LOAD_PATH")
> "")))
> + (setenv "JULIA_CPU_THREADS" jobs)
> (setenv "HOME" "/tmp")
> (invoke "julia" "--depwarn=yes"
> + (string-append "--procs=" jobs)
> (string-append builddir "loadpath/"
> package "/test/runtests.jl"))))
> #t)
I've applied this series as of commit
905b090582e5c03ef4fb183fda0a1ababf5cb284. Thanks for the great hacking
session (and result :-)).
Closing.
Maxim
bug archived.
Request was from
Debbugs Internal Request <help-debbugs <at> gnu.org>
to
internal_control <at> debbugs.gnu.org
.
(Thu, 23 Dec 2021 12:24:09 GMT)
Full text and
rfc822 format available.
This bug report was last modified 2 years and 86 days ago.
Previous Next
GNU bug tracking system
Copyright (C) 1999 Darren O. Benham,
1997,2003 nCipher Corporation Ltd,
1994-97 Ian Jackson.