FSEvents backend for tup monitor

Tupfile

@@ -11,7 +11,11 @@ endif
 ifneq ($(TARGET),win32)
 client_objs += src/tup/vardict.o
 client_objs += src/tup/send_event.o
+ifeq ($(TARGET),macosx)
+client_objs += src/tup/flock/flock.o
+else
 client_objs += src/tup/flock/fcntl.o
+endif
 : $(client_objs) |> !ar |> libtup_client.a
 : src/tup/vardict.h |> !cp |> tup_client.h
 endif

Tuprules.tup

@@ -66,6 +66,7 @@ CC = gcc
 AR = ar
 
 TUP_MONITOR = null
+TUP_LOCKING = fcntl
 TUP_SUID_GROUP = root
 TUP_USE_SYSTEM_PCRE = y
 include $(TARGET).tup

build.sh

@@ -9,6 +9,9 @@ case "$os" in
 		default_server=fuse3
 		monitor=inotify.c
 	;;
+	Darwin)
+		monitor=fsevents.c
+	;;
 esac
 
 server=${TUP_SERVER:-$default_server}
@@ -44,9 +47,11 @@ case "$os" in
 	plat_cflags="$plat_cflags -D_REENTRANT"
 	;;
 	Darwin)
+	flock_backend=flock
 	plat_files="$plat_files ../src/compat/dummy.c"
 	plat_files="$plat_files ../src/compat/clearenv.c "
 	plat_cflags="$plat_cflags -include ../src/compat/macosx.h"
+	plat_ldflags="$plat_ldflags -framework CoreServices"
 	default_cc=clang
 	;;
 	FreeBSD)
@@ -62,6 +67,7 @@ case "$os" in
 	;;
 esac
 : ${CC:=$default_cc}
+: ${flock_backend:=fcntl}
 
 rm -rf build
 echo "  mkdir build"
@@ -87,7 +93,7 @@ CFLAGS="$CFLAGS -DTUP_SERVER=\"$server\""
 CFLAGS="$CFLAGS -DPCRE2_CODE_UNIT_WIDTH=8"
 CFLAGS="$CFLAGS -DHAVE_CONFIG_H"
 
-for i in ../src/tup/*.c ../src/tup/tup/main.c ../src/tup/monitor/$monitor ../src/tup/flock/fcntl.c ../src/inih/ini.c ../src/pcre/*.c $plat_files; do
+for i in ../src/tup/*.c ../src/tup/tup/main.c ../src/tup/monitor/$monitor ../src/tup/monitor/common.c ../src/tup/flock/$flock_backend.c ../src/inih/ini.c ../src/pcre/*.c $plat_files; do
 	echo "  bootstrap CC $CFLAGS $i"
 	# Put -I. first so we find our new luabuiltin.h file, not one built
 	# by a previous invocation of 'tup'.

macosx.tup

@@ -6,3 +6,6 @@ CFLAGS += -include compat/macosx.h
 TUP_SUID_GROUP = wheel
 
 TUP_SERVER = fuse
+TUP_MONITOR = fsevents
+TUP_LOCKING = flock
+LDFLAGS += -framework CoreServices

src/tup/db.c

@@ -49,7 +49,7 @@
 #include <sys/stat.h>
 #include "sqlite3/sqlite3.h"
 
-#define DB_VERSION 19
+#define DB_VERSION 20
 #define PARSER_VERSION 16
 
 enum {
@@ -71,6 +71,8 @@ enum {
 	DB_SET_FLAGS,
 	DB_SET_TYPE,
 	DB_SET_MTIME,
+	DB_SET_INUM,
+	DB_SELECT_BY_INUM,
 	DB_SET_SRCID,
 	DB_PRINT,
 	DB_REBUILD_ALL,
@@ -310,7 +312,7 @@ int tup_db_create(int db_sync, int memory_db)
 	int x;
 	const char *dbname;
 	const char *sql[] = {
-		"create table node (id integer primary key not null, dir integer not null, type integer not null, mtime integer not null, mtime_ns integer not null, srcid integer not null, name varchar(4096), display varchar(4096), flags varchar(256), unique(dir, name))",
+		"create table node (id integer primary key not null, dir integer not null, type integer not null, mtime integer not null, mtime_ns integer not null, srcid integer not null, name varchar(4096), display varchar(4096), flags varchar(256), inum integer not null default 0, unique(dir, name))",
 		"create table normal_link (from_id integer, to_id integer, unique(from_id, to_id))",
 		"create table sticky_link (from_id integer, to_id integer, unique(from_id, to_id))",
 		"create table group_link (from_id integer, to_id integer, cmdid integer, unique(from_id, to_id, cmdid))",
@@ -326,7 +328,7 @@ int tup_db_create(int db_sync, int memory_db)
 		"create index group_index2 on group_link(cmdid)",
 		"create index srcid_index on node(srcid)",
 		"insert into config values('db_version', 0)",
-		"insert into node values(1, 0, 2, -1, 0, -1, '.', NULL, NULL)",
+		"insert into node values(1, 0, 2, -1, 0, -1, '.', NULL, NULL, 0)",
 	};
 
 	if(memory_db) {
@@ -662,6 +664,13 @@ static int version_check(void)
 				"alter table node add column mtime_ns integer default 0",
 			}
 		},
+		{
+			/* Upgrade to version 20 */
+			"Added an inum column to record each node's inode so the scanner can detect renames (mv a b) instead of treating the moved entries as a delete+create. The macOS FSEvents monitor needs this — its bulk-rescan model cannot otherwise preserve TUP_NODE_GENERATED identity for files carried across a directory rename.",
+			{
+				"alter table node add column inum integer not null default 0",
+			}
+		},
 	};
 
 	if(tup_db_config_get_int("db_version", -1, &version) < 0)
@@ -2278,6 +2287,114 @@ int tup_db_set_mtime(struct tup_entry *tent, struct timespec mtime)
 	return 0;
 }
 
+int tup_db_set_inum(tupid_t tupid, ino_t inum)
+{
+	int rc;
+	sqlite3_stmt **stmt = &stmts[DB_SET_INUM];
+	static char s[] = "update node set inum=? where id=?";
+
+	transaction_check("%s [%llu, %lli]", s, (unsigned long long)inum, tupid);
+	if(!*stmt) {
+		if(sqlite3_prepare_v2(tup_db, s, sizeof(s), stmt, NULL) != 0) {
+			fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(tup_db));
+			fprintf(stderr, "Statement was: %s\n", s);
+			return -1;
+		}
+	}
+
+	if(sqlite3_bind_int64(*stmt, 1, (sqlite3_int64)inum) != 0) {
+		fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		return -1;
+	}
+	if(sqlite3_bind_int64(*stmt, 2, tupid) != 0) {
+		fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		return -1;
+	}
+
+	rc = sqlite3_step(*stmt);
+	if(msqlite3_reset(*stmt) != 0) {
+		fprintf(stderr, "SQL reset error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		return -1;
+	}
+	if(rc != SQLITE_DONE) {
+		fprintf(stderr, "SQL step error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		return -1;
+	}
+	return 0;
+}
+
+/* Look up a tup_entry by inode. Returns 0 with *tent set on success
+ * (or NULL if no match). Excludes inum=0 — that's the migration
+ * default for nodes whose inode hasn't been captured yet, and would
+ * otherwise produce a false rename-match against any unrecorded node.
+ *
+ * Scoped to TUP_NODE_FILE / TUP_NODE_GENERATED / TUP_NODE_DIR /
+ * TUP_NODE_GENERATED_DIR: those are the types the scanner stats and
+ * records inodes for. Other types (CMD, VAR, GHOST, GROUP, ROOT)
+ * have inum=0 by construction.
+ */
+int tup_db_select_tent_by_inum(ino_t inum, struct tup_entry **tent)
+{
+	int rc;
+	int dbrc;
+	sqlite3_stmt **stmt = &stmts[DB_SELECT_BY_INUM];
+	static char s[] = "select id from node where inum=? and (type=? or type=? or type=? or type=?) limit 1";
+	tupid_t tupid;
+
+	*tent = NULL;
+	if(inum == 0)
+		return 0;
+
+	transaction_check("%s [%llu]", s, (unsigned long long)inum);
+	if(!*stmt) {
+		if(sqlite3_prepare_v2(tup_db, s, sizeof(s), stmt, NULL) != 0) {
+			fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(tup_db));
+			fprintf(stderr, "Statement was: %s\n", s);
+			return -1;
+		}
+	}
+
+	if(sqlite3_bind_int64(*stmt, 1, (sqlite3_int64)inum) != 0) goto bind_err;
+	if(sqlite3_bind_int(*stmt, 2, TUP_NODE_FILE) != 0) goto bind_err;
+	if(sqlite3_bind_int(*stmt, 3, TUP_NODE_GENERATED) != 0) goto bind_err;
+	if(sqlite3_bind_int(*stmt, 4, TUP_NODE_DIR) != 0) goto bind_err;
+	if(sqlite3_bind_int(*stmt, 5, TUP_NODE_GENERATED_DIR) != 0) goto bind_err;
+
+	dbrc = sqlite3_step(*stmt);
+	if(dbrc == SQLITE_ROW) {
+		tupid = sqlite3_column_int64(*stmt, 0);
+		rc = 0;
+	} else if(dbrc == SQLITE_DONE) {
+		tupid = -1;
+		rc = 0;
+	} else {
+		fprintf(stderr, "SQL step error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		rc = -1;
+	}
+
+	if(msqlite3_reset(*stmt) != 0) {
+		fprintf(stderr, "SQL reset error: %s\n", sqlite3_errmsg(tup_db));
+		fprintf(stderr, "Statement was: %s\n", s);
+		return -1;
+	}
+	if(rc < 0)
+		return -1;
+	if(tupid >= 0) {
+		if(tup_entry_add(tupid, tent) < 0)
+			return -1;
+	}
+	return 0;
+bind_err:
+	fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db));
+	fprintf(stderr, "Statement was: %s\n", s);
+	return -1;
+}
+
 int tup_db_set_srcid(struct tup_entry *tent, tupid_t srcid)
 {
 	int rc;

src/tup/db.h

@@ -28,6 +28,7 @@
 #include "estring.h"
 #include <stdio.h>
 #include <time.h>
+#include <sys/types.h>
 
 #define TUP_CONFIG "tup.config"
 
@@ -98,6 +99,8 @@ int tup_db_set_display(struct tup_entry *tent, const char *display, int displayl
 int tup_db_set_flags(struct tup_entry *tent, const char *flags, int flagslen);
 int tup_db_set_type(struct tup_entry *tent, enum TUP_NODE_TYPE type);
 int tup_db_set_mtime(struct tup_entry *tent, struct timespec mtime);
+int tup_db_set_inum(tupid_t tupid, ino_t inum);
+int tup_db_select_tent_by_inum(ino_t inum, struct tup_entry **tent);
 int tup_db_set_srcid(struct tup_entry *tent, tupid_t srcid);
 int tup_db_normal_dir_to_generated(struct tup_entry *tent);
 int tup_db_print(FILE *stream, tupid_t tupid);

src/tup/flock/Tupfile

@@ -1,7 +1,3 @@
 include_rules
 
-ifneq ($(TARGET),win32)
-: foreach fcntl.c |> !cc |>
-else
-: foreach lock_file.c |> !cc |>
-endif
+: foreach $(TUP_LOCKING).c |> !cc |>

src/tup/flock/flock.c

@@ -0,0 +1,95 @@
+/* vim: set ts=8 sw=8 sts=8 noet tw=78:
+ *
+ * tup - A file-based build system
+ *
+ * Copyright (C) 2011-2026  Mike Shal <marfey@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* BSD flock(2) backend. Used on macOS because kqueue EVFILT_VNODE with
+ * NOTE_FUNLOCK can detect flock releases (but not fcntl lock releases),
+ * enabling event-driven lock coordination in the monitor.
+ */
+
+#define _ATFILE_SOURCE
+#include "tup/flock.h"
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/file.h>
+
+int tup_lock_open(int basefd, const char *lockname, tup_lock_t *lock)
+{
+	int fd;
+
+	fd = openat(basefd, lockname, O_RDWR | O_CREAT, 0666);
+	if(fd < 0) {
+		perror(lockname);
+		fprintf(stderr, "tup error: Unable to open lockfile.\n");
+		return -1;
+	}
+	*lock = fd;
+	return 0;
+}
+
+void tup_lock_close(tup_lock_t lock)
+{
+	if(close(lock) < 0) {
+		perror("close(lock)");
+	}
+}
+
+int tup_flock(tup_lock_t fd)
+{
+	if(flock(fd, LOCK_EX) < 0) {
+		perror("flock LOCK_EX");
+		return -1;
+	}
+	return 0;
+}
+
+/* Returns: -1 error, 0 got lock, 1 would block */
+int tup_try_flock(tup_lock_t fd)
+{
+	if(flock(fd, LOCK_EX | LOCK_NB) < 0) {
+		if(errno == EWOULDBLOCK)
+			return 1;
+		perror("flock LOCK_EX|LOCK_NB");
+		return -1;
+	}
+	return 0;
+}
+
+int tup_unflock(tup_lock_t fd)
+{
+	if(flock(fd, LOCK_UN) < 0) {
+		perror("flock LOCK_UN");
+		return -1;
+	}
+	return 0;
+}
+
+int tup_wait_flock(tup_lock_t fd)
+{
+	/* Not used on macOS - the fsevents monitor uses kqueue
+	 * NOTE_FUNLOCK/NOTE_ATTRIB for event-driven lock notification
+	 * instead of polling. It is not possible to achieve this API on
+	 * macOS without polling.
+	 */
+	(void)fd;
+	fprintf(stderr, "tup internal error: tup_wait_flock called on macOS\n");
+	return -1;
+}

src/tup/lock.c

@@ -123,6 +123,30 @@ void tup_lock_closeall(void)
 	tup_lock_close(sh_lock);
 }
 
+int tup_lock_reopen(void)
+{
+	/* After fork(), flock(2) locks are shared with the parent because
+	 * they are per-file-description. Re-open the lock files to get
+	 * independent file descriptions, then close the inherited ones.
+	 * Open first so we never lack an fd for any lock file.
+	 */
+	tup_lock_t old_sh = sh_lock;
+	tup_lock_t old_obj = obj_lock;
+	tup_lock_t old_tri = tri_lock;
+
+	if(tup_lock_open(tup_top_fd(), TUP_SHARED_LOCK, &sh_lock) < 0)
+		return -1;
+	if(tup_lock_open(tup_top_fd(), TUP_OBJECT_LOCK, &obj_lock) < 0)
+		return -1;
+	if(tup_lock_open(tup_top_fd(), TUP_TRI_LOCK, &tri_lock) < 0)
+		return -1;
+
+	tup_lock_close(old_sh);
+	tup_lock_close(old_obj);
+	tup_lock_close(old_tri);
+	return 0;
+}
+
 tup_lock_t tup_sh_lock(void)
 {
 	return sh_lock;

src/tup/lock.h

@@ -43,6 +43,12 @@ void tup_lock_exit(void);
 /** Just closes the locks. This should by called by any forked processes. */
 void tup_lock_closeall(void);
 
+/** Re-opens lock files after fork to get independent file descriptions.
+ * Required when using flock(2) because flock locks are per-file-description
+ * and are shared across fork.
+ */
+int tup_lock_reopen(void);
+
 /* Tri-lock functions */
 tup_lock_t tup_sh_lock(void);
 tup_lock_t tup_obj_lock(void);