Merge branch 'master' into fread_skip_none

Author: ben-schwen

.ci/README.md

@@ -12,7 +12,7 @@ Test jobs:
 - `test-lin-rel-cran` - `--as-cran` on Linux, strict test for final status of `R CMD check`.
 - `test-lin-dev-gcc-strict-cran` - `--as-cran` on Linux, `r-devel` built with `-enable-strict-barrier --disable-long-double`, test for compilation warnings, test for new NOTEs/WARNINGs from `R CMD check`.
 - `test-lin-dev-clang-cran` - same as `gcc-strict` job but R built with `clang` and  no `--enable-strict-barrier --disable-long-double` flags.
-- `test-lin-ancient-cran` - Stated R dependency version (currently 3.4.0) on Linux.
+- `test-lin-ancient-cran` - Stated R dependency version (currently 3.5.0) on Linux.
 - `test-lin-dev-clang-san` - `r-devel` on Linux built with `clang -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
 - `test-lin-dev-gcc-san` - `r-devel` on Linux built with `gcc -fsanitize=address,undefined` (including LeakSanitizer), test for sanitizer output in tests and examples.
 - `test-win-rel` - `r-release` on Windows.

.dev/CRAN_Release.cmd

@@ -47,12 +47,12 @@ rm -rf ./data.table.Rcheck
 
 checkbashisms ./configure  # for portability; e.g. Solaris 10 running Bourne shell; #3964
 
-# Ensure no non-ASCII, other than in README.md is ok
+# Ensure no non-ASCII, other than in README.md DESCRIPTION vignettes *.yml and *.po are ok 
 # tests.Rraw in particular have failed CRAN Solaris (only) due to this.
-grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -P -n "[\x80-\xFF]" ./
+grep -RI --exclude-dir=".git" --exclude-dir="vignettes" --exclude="*.yml" --exclude="*.md" --exclude="*.po" --exclude="*~" --exclude="DESCRIPTION" --color='auto' -P -n "[\x80-\xFF]" ./
 
 # Unicode is now ok. This unicode in tests.Rraw is passing on CRAN.
-grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -n "[\]u[0-9]" ./
+# grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -n "[\]u[0-9]" ./
 
 # Ensure no calls to omp_get_max_threads() also since access should be via getDTthreads()
 grep --exclude="./src/openmp-utils.c" omp_get_max_threads ./src/*
@@ -80,14 +80,14 @@ grep "Rprintf" ./src/init.c
 # workaround for IBM AIX - ensure no globals named 'nearest' or 'class'.
 # See https://github.com/Rdatatable/data.table/issues/1351
 grep "nearest *=" ./src/*.c  # none
-grep "class *=" ./src/*.c    # quite a few but none global
+grep "class *=" ./src/*.c    # only one
 
 # ensure no use of z_const from zconf.h; #3939
 grep "z_const" ./src/*.[hc]  # none other than the comment
 
 # No undefined type punning of the form:  *(long long *)&REAL(column)[i]
 # Failed clang 3.9.1 -O3 due to this, I think.
-grep "&REAL" ./src/*.c
+grep "&REAL" ./src/*.c  # one instance in init.c
 
 # No [UN]PROTECT_PTR, #3232
 grep "PROTECT_PTR" ./src/*.c
@@ -107,13 +107,13 @@ grep -n "[^A-Za-z0-9]F[^A-Za-z0-9]" ./inst/tests/tests.Rraw
 grep -Enr "\bifelse" R
 
 # use substr() instead of substring(), #4447
-grep -Fnr "substring" R
+grep -Fnr "substring" R  # one instance in test.data.table.R
 
 # No system.time in main tests.Rraw. Timings should be in benchmark.Rraw
-grep -Fn "system.time" ./inst/tests/*.Rraw | grep -Fv "benchmark.Rraw" | grep -Fv "this system.time usage ok"
+grep -Fn "system.time" ./inst/tests/*.Rraw | grep -Fv "benchmark.Rraw" | grep -Fv "this system.time usage ok"  # only in froll.Rraw
 
 # No tryCatch in *.Rraw -- tryCatch should be handled only in test() itself to avoid silently missed warnings/errors/output
-grep -Fn "tryCatch" ./inst/tests/*.Rraw
+grep -Fn "tryCatch" ./inst/tests/*.Rraw  # only one outside of test() ./inst/tests/tests.Rraw:21594
 
 # All % in *.Rd should be escaped otherwise text gets silently chopped
 grep -n "[^\]%" ./man/*.Rd
@@ -168,8 +168,8 @@ R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 # avoids many too-many-requests in --as-cran's ping-all-URLs step (20 mins) inside the `checking CRAN incoming feasibility...` step.
 # Many thanks to Dirk for the tipoff that setting this env variable solves the problem, #4832.
-R CMD check data.table_1.16.99.tar.gz --as-cran
-R CMD INSTALL data.table_1.16.99.tar.gz --html
+R CMD check data.table_1.18.99.tar.gz --as-cran
+R CMD INSTALL data.table_1.18.99.tar.gz --html
 
 # Test C locale doesn't break test suite (#2771)
 echo LC_ALL=C > ~/.Renviron
@@ -193,9 +193,9 @@ q("no")
 
 # User supplied PKG_CFLAGS and PKG_LIBS passed through, #4664
 # Next line from https://mac.r-project.org/openmp/. Should see the arguments passed through and then fail with gcc on linux.
-PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.16.99.tar.gz
+PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.18.99.tar.gz
 # Next line should work on Linux, just using superfluous and duplicate but valid parameters here to see them retained and work 
-PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.16.99.tar.gz
+PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.18.99.tar.gz
 
 R
 remove.packages("xml2")    # we checked the URLs; don't need to do it again (many minutes)
@@ -224,23 +224,23 @@ system.time(test.data.table(script="*.Rraw"))  # apx 8h = froll 3h + nafill 1m +
 
 
 ###############################################
-#  R 3.4.0 (stated dependency)
+#  R 3.5.0 (stated dependency)
 ###############################################
 
 ### ONE TIME BUILD
 sudo apt-get -y build-dep r-base
 cd ~/build
-wget http://cran.stat.ucla.edu/src/base/R-3/R-3.4.0.tar.gz
-tar xvf R-3.4.0.tar.gz
-cd R-3.4.0
+wget http://cran.stat.ucla.edu/src/base/R-3/R-3.5.0.tar.gz
+tar xvf R-3.5.0.tar.gz
+cd R-3.5.0
 CFLAGS="-fcommon" FFLAGS="-fallow-argument-mismatch" ./configure --without-recommended-packages
 make
-alias R340=~/build/R-3.4.0/bin/R
+alias R350=~/build/R-3.5.0/bin/R
 ### END ONE TIME BUILD
 
 cd ~/GitHub/data.table
-R340 CMD INSTALL ./data.table_1.16.99.tar.gz
-R340
+R350 CMD INSTALL ./data.table_1.18.99.tar.gz
+R350
 require(data.table)
 test.data.table(script="*.Rraw")
 
@@ -251,15 +251,15 @@ test.data.table(script="*.Rraw")
 vi ~/.R/Makevars
 # Make line SHLIB_OPENMP_CFLAGS= active to remove -fopenmp
 R CMD build .
-R CMD INSTALL data.table_1.16.99.tar.gz   # ensure that -fopenmp is missing and there are no warnings
+R CMD INSTALL data.table_1.18.99.tar.gz   # ensure that -fopenmp is missing and there are no warnings
 R
 require(data.table)   # observe startup message about no OpenMP detected
 test.data.table()
 q("no")
 vi ~/.R/Makevars
 # revert change above
 R CMD build .
-R CMD check data.table_1.16.99.tar.gz
+R CMD check data.table_1.18.99.tar.gz
 
 
 #####################################################
@@ -568,7 +568,7 @@ du -k inst/tests                # 0.75MiB after
 R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 Rdevel -q -e "packageVersion('xml2')"   # ensure installed
-Rdevel CMD check data.table_1.17.0.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
+Rdevel CMD check data.table_1.18.0.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
 bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit compressed *.Rraw to git
 
 #
@@ -596,12 +596,12 @@ bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit com
 # 0. Start a new branch `cran-x.y.0` with the code as submitted to CRAN
 #    - Check that 'git status' shows 4 files in modified and uncommitted state: DESCRIPTION, NEWS.md, init.c and this .dev/CRAN_Release.cmd
 #    - The branch should have one commit with precisely these 4 files being edited
-# 1. Follow up with a commit with this consistent commit message like: "1.17.0 on CRAN. Bump to 1.17.99" to this branch bumping to the next dev version
+# 1. Follow up with a commit with this consistent commit message like: "1.18.0 on CRAN. Bump to 1.18.99" to this branch bumping to the next dev version
 #    - Bump minor version in DESCRIPTION to next odd number. Note that DESCRIPTION was in edited and uncommitted state so even number never appears in git.
 #    - Add new heading in NEWS for the next dev version. Add "(submitted to CRAN on <today>)" on the released heading.
 #    - Bump minor version in dllVersion() in init.c
 #    - Bump 3 minor version numbers in Makefile
-#    - Search and replace this .dev/CRAN_Release.cmd to update 1.16.99 to 1.16.99 inc below, 1.16.0 to 1.17.0 above, 1.15.0 to 1.16.0 below
+#    - Search and replace this .dev/CRAN_Release.cmd to update 1.17.99 to 1.18.99 inc below, 1.17.0 to 1.18.0
 #    - Another final gd to view all diffs using meld. (I have `alias gd='git difftool &> /dev/null'` and difftool meld: http://meldmerge.org/)
 # 2. Ideally, no PRs are reviewed while a CRAN submission is pending. Any reviews that do happen MUST target this branch, NOT master!
 # 3. Once the submission lands on CRAN, merge this branch WITHOUT SQUASHING!

.devcontainer/r-ancient-gcc/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.gitlab.com/rdatatable/dockerfiles/r-3.4.0
+FROM registry.gitlab.com/rdatatable/dockerfiles/r-3.5.0
 
 RUN apt-get -qq update \
   && apt-get install -y --no-install-recommends git

.github/workflows/R-CMD-check-occasional.yaml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macOS-latest, windows-latest, ubuntu-latest]
-        r: ['devel', 'release', '3.4', '3.5', '3.6', '4.0', '4.1', '4.2', '4.3']
+        r: ['devel', 'release', '3.5', '3.6', '4.0', '4.1', '4.2', '4.3']
         locale: ['en_US.utf8', 'zh_CN.utf8', 'lv_LV.utf8'] # Chinese for translations, Latvian for collate order (#3502)
         exclude:
           # only run non-English locale CI on Ubuntu
@@ -29,8 +29,6 @@ jobs:
           - os: windows-latest
             locale: 'lv_LV.utf8'
           # macOS/arm64 only available for R>=4.1.0
-          - os: macOS-latest
-            r: '3.4'
           - os: macOS-latest
             r: '3.5'
           - os: macOS-latest

.gitlab-ci.yml

@@ -10,8 +10,7 @@ variables:
   _R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"
   _R_CHECK_SYSTEM_CLOCK_: "false"  ## https://stackoverflow.com/questions/63613301/r-cmd-check-note-unable-to-verify-current-time
   _R_CHECK_TESTS_NLINES_: "0"
-  TZ: "UTC"  ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4.
-             ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
+  TZ: "UTC"  ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
              ## a non-UTC timezone, although, that's what we do routinely in dev.
   RUN_ALL_DATATABLE_TESTS: "yes"  ## run optional tests in CI
   R_REL_VERSION:  "4.5" # only raise when RTOOLS for REL is available
@@ -49,9 +48,6 @@ mirror-packages:
   cache:
     paths:
       - bus/$CI_JOB_NAME/cran
-  variables:
-    # TODO(R-ancient>=3.5.0): remove this; let it save PACKAGES.rds in version-3 format
-    R_DEFAULT_SERIALIZE_VERSION: 2
   script:
     - echo 'source(".ci/ci.R")' >> .Rprofile
     - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
@@ -211,7 +207,7 @@ test-lin-dev-clang-cran:
 # stated dependency on R
 test-lin-ancient-cran:
   <<: *test-lin
-  image: registry.gitlab.com/rdatatable/dockerfiles/r-3.4.0
+  image: registry.gitlab.com/rdatatable/dockerfiles/r-3.5.0
   variables:
     _R_CHECK_FORCE_SUGGESTS_: "FALSE" # can be removed if all dependencies are available (knitr, xts, etc.)
   script:

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: data.table
-Version: 1.17.99
+Version: 1.18.99
 Title: Extension of `data.frame`
-Depends: R (>= 3.4.0)
+Depends: R (>= 3.5.0)
 Imports: methods
 Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), R.utils (>= 2.13.0), xts, zoo (>= 1.8-1), yaml, knitr, markdown
 Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.

Makefile

@@ -18,7 +18,7 @@ some:
 
 .PHONY: clean
 clean:
-	$(RM) data.table_1.17.99.tar.gz
+	$(RM) data.table_1.18.99.tar.gz
 	$(RM) src/*.o
 	$(RM) src/*.so
 
@@ -28,7 +28,7 @@ build:
 
 .PHONY: install
 install:
-	$(R) CMD INSTALL data.table_1.17.99.tar.gz
+	$(R) CMD INSTALL data.table_1.18.99.tar.gz
 
 .PHONY: uninstall
 uninstall:
@@ -40,7 +40,7 @@ test:
 
 .PHONY: check
 check:
-	_R_CHECK_CRAN_INCOMING_REMOTE_=false $(R) CMD check data.table_1.17.99.tar.gz --as-cran --ignore-vignettes --no-stop-on-test-error
+	_R_CHECK_CRAN_INCOMING_REMOTE_=false $(R) CMD check data.table_1.18.99.tar.gz --as-cran --ignore-vignettes --no-stop-on-test-error
 
 .PHONY: revision
 revision:

NEWS.md

@@ -2,7 +2,17 @@
 
 **If you are viewing this file on CRAN, please check [latest news on GitHub](https://github.com/Rdatatable/data.table/blob/master/NEWS.md) where the formatting is also better.**
 
-## data.table [v1.17.99](https://github.com/Rdatatable/data.table/milestone/35)  (in development)
+## data.table [v1.18.99](https://github.com/Rdatatable/data.table/milestone/37?closed=1)  (in development)
+
+### Notes
+
+1. {data.table} now depends on R 3.5.0 (2018).
+
+### BUG FIXES
+
+1. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix.
+
+## data.table [v1.18.0](https://github.com/Rdatatable/data.table/milestone/37?closed=1)  23 December 2025
 
 ### BREAKING CHANGE
 
@@ -11,6 +21,7 @@
 2. `melt()` returns an integer column for `variable` when `measure.vars` is a list of length=1, consistent with the documented behavior, [#5209](https://github.com/Rdatatable/data.table/issues/5209). Thanks to @tdhock for reporting. Any users who were relying on this behavior can change `measure.vars=list("col_name")` (output `variable` was column name, now is column index/integer) to `measure.vars="col_name"` (`variable` still is column name). This change has been planned since 1.16.0 (25 Aug 2024).
 
 3. Rolling functions `frollmean` and `frollsum` distinguish `Inf`/`-Inf` from `NA` to match the same rules as base R when `algo="fast"` (previously they were considered the same). If your input into those functions has `Inf` or `-Inf` then you will be affected by this change. As a result, the argument that controls the handling of `NA`s has been renamed from `hasNA` to `has.nf` (_has non-finite_). `hasNA` continues to work with a warning, for now.
+
     ```r
     ## before
     frollsum(c(1,2,3,Inf,5,6), 2)
@@ -19,8 +30,10 @@
     ## now
     frollsum(c(1,2,3,Inf,5,6), 2)
     #[1]  NA   3   5 Inf Inf  11
+    ```
 
 4. `frollapply` result is not coerced to numeric anymore. Users' code could possibly break if it depends on forced coercion of input/output to numeric type.
+
     ```r
     ## before
     frollapply(c(F,T,F,F,F,T), 2, any)
@@ -30,6 +43,7 @@
     frollapply(c(F,T,F,F,F,T), 2, any)
     #[1]    NA  TRUE  TRUE FALSE FALSE  TRUE
     ```
+
     Additionally argument names in `frollapply` has been renamed from `x` to `X` and `n` to `N` to avoid conflicts with common argument names that may be passed to `...`, aligning to base R API of `lapply`. `x` and `n` continue to work with a warning, for now.
 
 5. Negative and missing values of `n` argument of adaptive rolling functions trigger an error.
@@ -218,6 +232,7 @@ See [#2611](https://github.com/Rdatatable/data.table/issues/2611) for details. T
     ```
 
 18. New helper `frolladapt` to facilitate applying rolling functions over windows of fixed calendar-time width in irregularly-spaced data sets, thereby bypassing the need to "augment" such data with placeholder rows, [#3241](https://github.com/Rdatatable/data.table/issues/3241). Thanks to @jangorecki for implementation.
+
     ```r
     idx = as.Date("2025-09-05") + c(0,4,7,8,9,10,12,13,17)
     dt = data.table(index=idx, value=seq_along(idx))
@@ -354,7 +369,7 @@ See [#2611](https://github.com/Rdatatable/data.table/issues/2611) for details. T
 
 27. `dogroups()` no longer reads beyond the resized end of over-allocated data.table list columns, [#7486](https://github.com/Rdatatable/data.table/issues/7486). While this didn't crash in practice, it is now explicitly checked for in recent R versions (r89198+). Thanks @TimTaylor and @aitap for the report and @aitap for the fix.
 
-28. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix.
+28. `rbindlist()` now avoids the crash when working with many non-UTF-8 column names, [#7452](https://github.com/Rdatatable/data.table/issues/7452). Thanks @aitap for the report and the fix.
 
 ### NOTES
 

R/IDateTime.R

@@ -124,9 +124,7 @@ chooseOpsMethod.IDate = function(x, y, mx, my, cl, reverse) inherits(y, "Date")
   if ( is.double(e2) && !fitsInInt32(e2) ) {
     # IDate deliberately doesn't support fractional days so revert to base Date
     return(base::`-.Date`(as.Date(e1), e2))
-    # can't call base::.Date directly (last line of base::`-.Date`) as tried in PR#3168 because
-    # i) ?.Date states "Internal objects in the base package most of which are only user-visible because of the special nature of the base namespace."
-    # ii) .Date was newly exposed in R some time after 3.4.4
+    # can't call base::.Date directly (last line of base::`-.Date`) as tried in PR#3168 because ?.Date states "Internal objects in the base package most of which are only user-visible because of the special nature of the base namespace."
   }
   ans = as.integer(unclass(e1) - unclass(e2))
   if (inherits(e2, "Date")) {

R/as.data.table.R

@@ -102,8 +102,9 @@ as.data.table.array = function(x, keep.rownames=FALSE, key=NULL, sorted=TRUE, va
     lapply(dx, seq_len)
   } else if (any(nulldnx <- vapply_1b(dnx, is.null))) {
     dnx[nulldnx] = lapply(dx[nulldnx], seq_len) #3636
+    setattr(dnx, 'names', copy(names(dnx)))
     dnx
-  } else dnx
+  } else copy(dnx)
   setfrev(val)
   if (is.null(names(val)) || !any(nzchar(names(val))))
     setattr(val, 'names', paste0("V", frev(seq_along(val))))