diff --git a/Tupfile b/Tupfile index dad10c925..ed00869ed 100644 --- a/Tupfile +++ b/Tupfile @@ -11,7 +11,11 @@ endif ifneq ($(TARGET),win32) client_objs += src/tup/vardict.o client_objs += src/tup/send_event.o +ifeq ($(TARGET),macosx) +client_objs += src/tup/flock/flock.o +else client_objs += src/tup/flock/fcntl.o +endif : $(client_objs) |> !ar |> libtup_client.a : src/tup/vardict.h |> !cp |> tup_client.h endif diff --git a/Tuprules.tup b/Tuprules.tup index 3b774373e..e8df02ee7 100644 --- a/Tuprules.tup +++ b/Tuprules.tup @@ -66,6 +66,7 @@ CC = gcc AR = ar TUP_MONITOR = null +TUP_LOCKING = fcntl TUP_SUID_GROUP = root TUP_USE_SYSTEM_PCRE = y include $(TARGET).tup diff --git a/build.sh b/build.sh index 62bb362a6..5d8f242da 100755 --- a/build.sh +++ b/build.sh @@ -9,6 +9,9 @@ case "$os" in default_server=fuse3 monitor=inotify.c ;; + Darwin) + monitor=fsevents.c + ;; esac server=${TUP_SERVER:-$default_server} @@ -44,9 +47,11 @@ case "$os" in plat_cflags="$plat_cflags -D_REENTRANT" ;; Darwin) + flock_backend=flock plat_files="$plat_files ../src/compat/dummy.c" plat_files="$plat_files ../src/compat/clearenv.c " plat_cflags="$plat_cflags -include ../src/compat/macosx.h" + plat_ldflags="$plat_ldflags -framework CoreServices" default_cc=clang ;; FreeBSD) @@ -62,6 +67,7 @@ case "$os" in ;; esac : ${CC:=$default_cc} +: ${flock_backend:=fcntl} rm -rf build echo " mkdir build" @@ -87,7 +93,7 @@ CFLAGS="$CFLAGS -DTUP_SERVER=\"$server\"" CFLAGS="$CFLAGS -DPCRE2_CODE_UNIT_WIDTH=8" CFLAGS="$CFLAGS -DHAVE_CONFIG_H" -for i in ../src/tup/*.c ../src/tup/tup/main.c ../src/tup/monitor/$monitor ../src/tup/flock/fcntl.c ../src/inih/ini.c ../src/pcre/*.c $plat_files; do +for i in ../src/tup/*.c ../src/tup/tup/main.c ../src/tup/monitor/$monitor ../src/tup/monitor/common.c ../src/tup/flock/$flock_backend.c ../src/inih/ini.c ../src/pcre/*.c $plat_files; do echo " bootstrap CC $CFLAGS $i" # Put -I. first so we find our new luabuiltin.h file, not one built # by a previous invocation of 'tup'. diff --git a/macosx.tup b/macosx.tup index f32a99953..e3ff7aa46 100644 --- a/macosx.tup +++ b/macosx.tup @@ -6,3 +6,6 @@ CFLAGS += -include compat/macosx.h TUP_SUID_GROUP = wheel TUP_SERVER = fuse +TUP_MONITOR = fsevents +TUP_LOCKING = flock +LDFLAGS += -framework CoreServices diff --git a/src/tup/db.c b/src/tup/db.c index 4c99b2ce4..90cf1e890 100644 --- a/src/tup/db.c +++ b/src/tup/db.c @@ -49,7 +49,7 @@ #include #include "sqlite3/sqlite3.h" -#define DB_VERSION 19 +#define DB_VERSION 20 #define PARSER_VERSION 16 enum { @@ -71,6 +71,8 @@ enum { DB_SET_FLAGS, DB_SET_TYPE, DB_SET_MTIME, + DB_SET_INUM, + DB_SELECT_BY_INUM, DB_SET_SRCID, DB_PRINT, DB_REBUILD_ALL, @@ -310,7 +312,7 @@ int tup_db_create(int db_sync, int memory_db) int x; const char *dbname; const char *sql[] = { - "create table node (id integer primary key not null, dir integer not null, type integer not null, mtime integer not null, mtime_ns integer not null, srcid integer not null, name varchar(4096), display varchar(4096), flags varchar(256), unique(dir, name))", + "create table node (id integer primary key not null, dir integer not null, type integer not null, mtime integer not null, mtime_ns integer not null, srcid integer not null, name varchar(4096), display varchar(4096), flags varchar(256), inum integer not null default 0, unique(dir, name))", "create table normal_link (from_id integer, to_id integer, unique(from_id, to_id))", "create table sticky_link (from_id integer, to_id integer, unique(from_id, to_id))", "create table group_link (from_id integer, to_id integer, cmdid integer, unique(from_id, to_id, cmdid))", @@ -326,7 +328,7 @@ int tup_db_create(int db_sync, int memory_db) "create index group_index2 on group_link(cmdid)", "create index srcid_index on node(srcid)", "insert into config values('db_version', 0)", - "insert into node values(1, 0, 2, -1, 0, -1, '.', NULL, NULL)", + "insert into node values(1, 0, 2, -1, 0, -1, '.', NULL, NULL, 0)", }; if(memory_db) { @@ -662,6 +664,13 @@ static int version_check(void) "alter table node add column mtime_ns integer default 0", } }, + { + /* Upgrade to version 20 */ + "Added an inum column to record each node's inode so the scanner can detect renames (mv a b) instead of treating the moved entries as a delete+create. The macOS FSEvents monitor needs this — its bulk-rescan model cannot otherwise preserve TUP_NODE_GENERATED identity for files carried across a directory rename.", + { + "alter table node add column inum integer not null default 0", + } + }, }; if(tup_db_config_get_int("db_version", -1, &version) < 0) @@ -2278,6 +2287,114 @@ int tup_db_set_mtime(struct tup_entry *tent, struct timespec mtime) return 0; } +int tup_db_set_inum(tupid_t tupid, ino_t inum) +{ + int rc; + sqlite3_stmt **stmt = &stmts[DB_SET_INUM]; + static char s[] = "update node set inum=? where id=?"; + + transaction_check("%s [%llu, %lli]", s, (unsigned long long)inum, tupid); + if(!*stmt) { + if(sqlite3_prepare_v2(tup_db, s, sizeof(s), stmt, NULL) != 0) { + fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + } + + if(sqlite3_bind_int64(*stmt, 1, (sqlite3_int64)inum) != 0) { + fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + if(sqlite3_bind_int64(*stmt, 2, tupid) != 0) { + fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + + rc = sqlite3_step(*stmt); + if(msqlite3_reset(*stmt) != 0) { + fprintf(stderr, "SQL reset error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + if(rc != SQLITE_DONE) { + fprintf(stderr, "SQL step error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + return 0; +} + +/* Look up a tup_entry by inode. Returns 0 with *tent set on success + * (or NULL if no match). Excludes inum=0 — that's the migration + * default for nodes whose inode hasn't been captured yet, and would + * otherwise produce a false rename-match against any unrecorded node. + * + * Scoped to TUP_NODE_FILE / TUP_NODE_GENERATED / TUP_NODE_DIR / + * TUP_NODE_GENERATED_DIR: those are the types the scanner stats and + * records inodes for. Other types (CMD, VAR, GHOST, GROUP, ROOT) + * have inum=0 by construction. + */ +int tup_db_select_tent_by_inum(ino_t inum, struct tup_entry **tent) +{ + int rc; + int dbrc; + sqlite3_stmt **stmt = &stmts[DB_SELECT_BY_INUM]; + static char s[] = "select id from node where inum=? and (type=? or type=? or type=? or type=?) limit 1"; + tupid_t tupid; + + *tent = NULL; + if(inum == 0) + return 0; + + transaction_check("%s [%llu]", s, (unsigned long long)inum); + if(!*stmt) { + if(sqlite3_prepare_v2(tup_db, s, sizeof(s), stmt, NULL) != 0) { + fprintf(stderr, "SQL Error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + } + + if(sqlite3_bind_int64(*stmt, 1, (sqlite3_int64)inum) != 0) goto bind_err; + if(sqlite3_bind_int(*stmt, 2, TUP_NODE_FILE) != 0) goto bind_err; + if(sqlite3_bind_int(*stmt, 3, TUP_NODE_GENERATED) != 0) goto bind_err; + if(sqlite3_bind_int(*stmt, 4, TUP_NODE_DIR) != 0) goto bind_err; + if(sqlite3_bind_int(*stmt, 5, TUP_NODE_GENERATED_DIR) != 0) goto bind_err; + + dbrc = sqlite3_step(*stmt); + if(dbrc == SQLITE_ROW) { + tupid = sqlite3_column_int64(*stmt, 0); + rc = 0; + } else if(dbrc == SQLITE_DONE) { + tupid = -1; + rc = 0; + } else { + fprintf(stderr, "SQL step error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + rc = -1; + } + + if(msqlite3_reset(*stmt) != 0) { + fprintf(stderr, "SQL reset error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; + } + if(rc < 0) + return -1; + if(tupid >= 0) { + if(tup_entry_add(tupid, tent) < 0) + return -1; + } + return 0; +bind_err: + fprintf(stderr, "SQL bind error: %s\n", sqlite3_errmsg(tup_db)); + fprintf(stderr, "Statement was: %s\n", s); + return -1; +} + int tup_db_set_srcid(struct tup_entry *tent, tupid_t srcid) { int rc; diff --git a/src/tup/db.h b/src/tup/db.h index c9f1c9db0..f93817ef3 100644 --- a/src/tup/db.h +++ b/src/tup/db.h @@ -28,6 +28,7 @@ #include "estring.h" #include #include +#include #define TUP_CONFIG "tup.config" @@ -98,6 +99,8 @@ int tup_db_set_display(struct tup_entry *tent, const char *display, int displayl int tup_db_set_flags(struct tup_entry *tent, const char *flags, int flagslen); int tup_db_set_type(struct tup_entry *tent, enum TUP_NODE_TYPE type); int tup_db_set_mtime(struct tup_entry *tent, struct timespec mtime); +int tup_db_set_inum(tupid_t tupid, ino_t inum); +int tup_db_select_tent_by_inum(ino_t inum, struct tup_entry **tent); int tup_db_set_srcid(struct tup_entry *tent, tupid_t srcid); int tup_db_normal_dir_to_generated(struct tup_entry *tent); int tup_db_print(FILE *stream, tupid_t tupid); diff --git a/src/tup/flock/Tupfile b/src/tup/flock/Tupfile index 7f1d54415..96a4ca9cd 100644 --- a/src/tup/flock/Tupfile +++ b/src/tup/flock/Tupfile @@ -1,7 +1,3 @@ include_rules -ifneq ($(TARGET),win32) -: foreach fcntl.c |> !cc |> -else -: foreach lock_file.c |> !cc |> -endif +: foreach $(TUP_LOCKING).c |> !cc |> diff --git a/src/tup/flock/flock.c b/src/tup/flock/flock.c new file mode 100644 index 000000000..ac7688526 --- /dev/null +++ b/src/tup/flock/flock.c @@ -0,0 +1,95 @@ +/* vim: set ts=8 sw=8 sts=8 noet tw=78: + * + * tup - A file-based build system + * + * Copyright (C) 2011-2026 Mike Shal + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* BSD flock(2) backend. Used on macOS because kqueue EVFILT_VNODE with + * NOTE_FUNLOCK can detect flock releases (but not fcntl lock releases), + * enabling event-driven lock coordination in the monitor. + */ + +#define _ATFILE_SOURCE +#include "tup/flock.h" +#include +#include +#include +#include +#include + +int tup_lock_open(int basefd, const char *lockname, tup_lock_t *lock) +{ + int fd; + + fd = openat(basefd, lockname, O_RDWR | O_CREAT, 0666); + if(fd < 0) { + perror(lockname); + fprintf(stderr, "tup error: Unable to open lockfile.\n"); + return -1; + } + *lock = fd; + return 0; +} + +void tup_lock_close(tup_lock_t lock) +{ + if(close(lock) < 0) { + perror("close(lock)"); + } +} + +int tup_flock(tup_lock_t fd) +{ + if(flock(fd, LOCK_EX) < 0) { + perror("flock LOCK_EX"); + return -1; + } + return 0; +} + +/* Returns: -1 error, 0 got lock, 1 would block */ +int tup_try_flock(tup_lock_t fd) +{ + if(flock(fd, LOCK_EX | LOCK_NB) < 0) { + if(errno == EWOULDBLOCK) + return 1; + perror("flock LOCK_EX|LOCK_NB"); + return -1; + } + return 0; +} + +int tup_unflock(tup_lock_t fd) +{ + if(flock(fd, LOCK_UN) < 0) { + perror("flock LOCK_UN"); + return -1; + } + return 0; +} + +int tup_wait_flock(tup_lock_t fd) +{ + /* Not used on macOS - the fsevents monitor uses kqueue + * NOTE_FUNLOCK/NOTE_ATTRIB for event-driven lock notification + * instead of polling. It is not possible to achieve this API on + * macOS without polling. + */ + (void)fd; + fprintf(stderr, "tup internal error: tup_wait_flock called on macOS\n"); + return -1; +} diff --git a/src/tup/lock.c b/src/tup/lock.c index d2f93c231..42a5ea265 100644 --- a/src/tup/lock.c +++ b/src/tup/lock.c @@ -123,6 +123,30 @@ void tup_lock_closeall(void) tup_lock_close(sh_lock); } +int tup_lock_reopen(void) +{ + /* After fork(), flock(2) locks are shared with the parent because + * they are per-file-description. Re-open the lock files to get + * independent file descriptions, then close the inherited ones. + * Open first so we never lack an fd for any lock file. + */ + tup_lock_t old_sh = sh_lock; + tup_lock_t old_obj = obj_lock; + tup_lock_t old_tri = tri_lock; + + if(tup_lock_open(tup_top_fd(), TUP_SHARED_LOCK, &sh_lock) < 0) + return -1; + if(tup_lock_open(tup_top_fd(), TUP_OBJECT_LOCK, &obj_lock) < 0) + return -1; + if(tup_lock_open(tup_top_fd(), TUP_TRI_LOCK, &tri_lock) < 0) + return -1; + + tup_lock_close(old_sh); + tup_lock_close(old_obj); + tup_lock_close(old_tri); + return 0; +} + tup_lock_t tup_sh_lock(void) { return sh_lock; diff --git a/src/tup/lock.h b/src/tup/lock.h index f99a6c487..d967510d4 100644 --- a/src/tup/lock.h +++ b/src/tup/lock.h @@ -43,6 +43,12 @@ void tup_lock_exit(void); /** Just closes the locks. This should by called by any forked processes. */ void tup_lock_closeall(void); +/** Re-opens lock files after fork to get independent file descriptions. + * Required when using flock(2) because flock locks are per-file-description + * and are shared across fork. + */ +int tup_lock_reopen(void); + /* Tri-lock functions */ tup_lock_t tup_sh_lock(void); tup_lock_t tup_obj_lock(void); diff --git a/src/tup/monitor/Tupfile b/src/tup/monitor/Tupfile index 2393e9ea6..e12f2e9a8 100644 --- a/src/tup/monitor/Tupfile +++ b/src/tup/monitor/Tupfile @@ -1,3 +1,3 @@ include_rules -: $(TUP_MONITOR).c |> !cc |> +: foreach $(TUP_MONITOR).c common.c |> !cc |> diff --git a/src/tup/monitor/common.c b/src/tup/monitor/common.c new file mode 100644 index 000000000..d49b3ce46 --- /dev/null +++ b/src/tup/monitor/common.c @@ -0,0 +1,348 @@ +/* vim: set ts=8 sw=8 sts=8 noet tw=78: + * + * tup - A file-based build system + * + * Copyright (C) 2008-2026 Mike Shal + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* Shared monitor utilities used by both the inotify and fsevents backends. */ + +#define _ATFILE_SOURCE +#include "tup/monitor.h" +#include "monitor_common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tup/config.h" +#include "tup/db.h" +#include "tup/debug.h" +#include "tup/flock.h" +#include "tup/fslurp.h" +#include "tup/option.h" + +char **update_argv; +int update_argc; +int autoupdate_flag = -1; +int autoparse_flag = -1; +pthread_mutex_t autoupdate_lock = PTHREAD_MUTEX_INITIALIZER; +pthread_cond_t autoupdate_cond = PTHREAD_COND_INITIALIZER; +pid_t autoupdate_pid = AUTOUPDATE_NONE; + +/* Arguments are cleared to "-" if they are used by the monitor. These + * args are also passed on to the autoupdate process if that feature is + * enabled, but we don't want the updater getting any args that are + * meant for the monitor. Ultimately the options may end up at + * prune_graph(), which ignores args that begin with '-'. + */ +void monitor_parse_args(int argc, char **argv, int *foreground) +{ + int x; + + *foreground = tup_option_get_flag("monitor.foreground"); + + for(x=0; x= (signed)sizeof(buf)) { + fprintf(stderr, "Buf is sized too small in monitor_set_pid\n"); + return -1; + } + if(write(fd, buf, len) < 0) { + perror("write"); + return -1; + } + if(ftruncate(fd, len) < 0) { + perror("ftruncate"); + return -1; + } + if(tup_unflock(fd) < 0) { + return -1; + } + if(close(fd) < 0) { + perror("close(fd"); + return -1; + } + return 0; +} + +int monitor_get_pid(int restarting, int *pid) +{ + struct buf b; + int fd; + + *pid = -1; + fd = openat(tup_top_fd(), MONITOR_PID_FILE, O_RDWR, 0666); + if(fd < 0) { + if(errno != ENOENT) { + perror(MONITOR_PID_FILE); + return -1; + } + /* No pid file means we don't have the monitor running, so just + * leave it at -1 and return success. + */ + return 0; + } + if(tup_flock(fd) < 0) { + return -1; + } + if(fslurp_null(fd, &b) < 0) { + goto out; + } + + if(b.len > 0) { + *pid = strtol(b.s, NULL, 0); + } + free(b.s); +out: + if(tup_unflock(fd) < 0) { + return -1; + } + if(close(fd) < 0) { + perror("close(fd"); + return -1; + } + + if(*pid > 0) { + /* Just using getpriority() to see if the monitor process is + * alive. + */ + errno = 0; + if(getpriority(PRIO_PROCESS, *pid) == -1 && errno == ESRCH) { + printf("Monitor pid %i doesn't exist anymore.\n", *pid); + if(restarting == TUP_MONITOR_RESTARTING) { + /* If we are actually restarting the monitor + * make sure we let them know that the 'pid + * doesn't exist anymore' message isn't just + * an error message. + */ + printf("Restarting the monitor.\n"); + } + monitor_set_pid(-1); + *pid = -1; + } + } + return 0; +} + +int stop_monitor(int restarting) +{ + int pid; + + if(monitor_get_pid(restarting, &pid) < 0) { + fprintf(stderr, "tup error: Unable to get the current monitor pid in order to shut it down.\n"); + return -1; + } + if(pid < 0) { + if(restarting == TUP_MONITOR_SHUTDOWN) { + /* This case returns an error so we can tell in the + * test code if the monitor isn't actually running when + * it should be. + */ + printf("No monitor process to kill (pid < 0)\n"); + return -1; + } + return 0; + } + if(restarting == TUP_MONITOR_RESTARTING) + printf("Restarting the monitor.\n"); + else + printf("Shutting down the monitor.\n"); + if(kill(pid, SIGHUP) < 0) { + perror("kill"); + return -1; + } + + return 0; +} + +int autoupdate_enabled(void) +{ + int autoupdate_config; + if(autoupdate_flag == 1) + return 1; + autoupdate_config = tup_option_get_flag("monitor.autoupdate"); + if(autoupdate_flag == -1 && autoupdate_config == 1) + return 1; + return 0; +} + +int autoparse_enabled(void) +{ + int autoparse_config; + if(autoparse_flag == 1) + return 1; + autoparse_config = tup_option_get_flag("monitor.autoparse"); + if(autoparse_flag == -1 && autoparse_config == 1) + return 1; + return 0; +} + +int autoupdate(const char *cmd) +{ + /* This runs in a separate process (as opposed to just calling + * updater() directly) so it can properly get the lock from us (the + * monitor) and flush the queue correctly. Otherwise files touched by + * the updater will be caught by us after we return to regular event + * processing mode, which is annoying. + */ + pid_t pid = fork(); + if(pid < 0) { + perror("fork"); + return -1; + } + if(pid == 0) { + char **args; + int x; + + args = malloc((sizeof *args) * (update_argc + 4)); + if(!args) { + perror("malloc"); + exit(1); + } + args[0] = strdup("tup"); + if(!args[0]) { + perror("strdup"); + exit(1); + } + args[1] = strdup(cmd); + if(!args[1]) { + perror("strdup"); + exit(1); + } + args[2] = strdup("--no-environ-check"); + if(!args[2]) { + perror("strdup"); + exit(1); + } + for(x=0; x + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* macOS FSEvents-based file monitor for tup. + * + * This uses FSEvents to watch the project directory tree for file changes. + * For lock coordination with other tup processes, we poll tup_try_flock() + * since kqueue EVFILT_VNODE cannot detect flock state changes on macOS. + */ + +#define _ATFILE_SOURCE +#include "tup/monitor.h" +#include "monitor_common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tup/debug.h" +#include "tup/fileio.h" +#include "tup/config.h" +#include "tup/db.h" +#include "tup/lock.h" +#include "tup/flock.h" +#include "tup/path.h" +#include "tup/entry.h" +#include "tup/fslurp.h" +#include "tup/server.h" +#include "tup/option.h" +#include "tup/timespan.h" +#include "tup/variant.h" +#include "tup/init.h" +#include "tup/pel_group.h" + +#define MONITOR_LOOP_RETRY -2 + +static int monitor_loop(void); +static int try_autoupdate(void); +static void sighandler(int sig); +static void fsevents_callback(ConstFSEventStreamRef streamRef, + void *clientCallBackInfo, + size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]); + +static struct sigaction sigact = { + .sa_handler = sighandler, + .sa_flags = 0, +}; +static volatile sig_atomic_t monitor_quit = 0; + +static int locked = 1; +static int backgrounding = 0; + +/* Tracks whether events occurred while locked (ie, while we have control) */ +static pthread_mutex_t event_lock = PTHREAD_MUTEX_INITIALIZER; +static int events_occurred = 0; + +int monitor_supported(void) +{ + return 0; +} + +int monitor(int argc, char **argv) +{ + int rc = 0; + int foreground; + pthread_t autoupdate_thread; + + /* Close down the fork process, since we don't need it. */ + if(server_post_exit() < 0) + return -1; + monitor_parse_args(argc, argv, &foreground); + + if(sigemptyset(&sigact.sa_mask) < 0) { + perror("sigemptyset"); + return -1; + } + if(sigaction(SIGINT, &sigact, NULL) < 0) { + perror("sigaction"); + return -1; + } + if(sigaction(SIGTERM, &sigact, NULL) < 0) { + perror("sigaction"); + return -1; + } + if(sigaction(SIGHUP, &sigact, NULL) < 0) { + perror("sigaction"); + return -1; + } + if(sigaction(SIGUSR1, &sigact, NULL) < 0) { + perror("sigaction"); + return -1; + } + + if(stop_monitor(TUP_MONITOR_RESTARTING) < 0) { + fprintf(stderr, "tup error: Unable to stop the current monitor process.\n"); + return -1; + } + + if(foreground) { + if(tup_unflock(tup_sh_lock()) < 0) { + return -1; + } + } else { + if(fork() > 0) { + int kq; + struct kevent ev; + + /* Set up kqueue to watch for the child's signal + * (a no-op ftruncate) on the obj-lock file. Register + * BEFORE unlocking so we can't miss the event. + */ + kq = kqueue(); + if(kq < 0) { + perror("kqueue"); + exit(1); + } + EV_SET(&ev, tup_obj_lock(), EVFILT_VNODE, + EV_ADD | EV_ENABLE, NOTE_ATTRIB, 0, NULL); + if(kevent(kq, &ev, 1, NULL, 0, NULL) < 0) { + perror("kevent register"); + close(kq); + exit(1); + } + + /* Release obj-lock, then wait for child to signal */ + tup_unflock(tup_obj_lock()); + if(kevent(kq, NULL, 0, &ev, 1, NULL) < 0) { + perror("kevent wait"); + close(kq); + exit(1); + } + close(kq); + + if(tup_cleanup() < 0) + exit(1); + tup_valgrind_cleanup(); + exit(0); + } + + /* Re-open lock files to get independent file descriptions. + * flock(2) locks are per-file-description and shared across + * fork, so we need our own fds to lock independently. + */ + if(tup_lock_reopen() < 0) { + return -1; + } + if(tup_flock(tup_obj_lock()) < 0) { + return -1; + } + backgrounding = 1; + } + + if(monitor_set_pid(getpid()) < 0) { + return -1; + } + + if(pthread_create(&autoupdate_thread, NULL, wait_thread, NULL) != 0) { + perror("pthread_create"); + return -1; + } + + do { + rc = monitor_loop(); + if(rc == MONITOR_LOOP_RETRY) { + /* Need to clear out all saved structures (the dircache + * and tup_entries), then shut the monitor off before + * turning it back on. If there is a waiting 'tup' + * it will get the lock and update in scan mode before + * we return from tup_lock_init(). Then we should be + * good to go. + */ + if(tup_entry_clear() < 0) + return -1; + if(monitor_set_pid(-1) < 0) + return -1; + tup_lock_closeall(); + + if(fchdir(tup_top_fd()) < 0) { + perror("fchdir tup_top"); + return -1; + } + if(tup_lock_init() < 0) + return -1; + + if(tup_unflock(tup_sh_lock()) < 0) { + return -1; + } + if(monitor_set_pid(getpid()) < 0) + return -1; + } + } while(rc == MONITOR_LOOP_RETRY); + monitor_set_pid(-1); + + pthread_mutex_lock(&autoupdate_lock); + autoupdate_pid = AUTOUPDATE_EXIT; + pthread_cond_signal(&autoupdate_cond); + pthread_mutex_unlock(&autoupdate_lock); + pthread_join(autoupdate_thread, NULL); + + return rc; +} + +static int mod_cb(void *arg, struct tup_entry *tent) +{ + /* Only user-visible file changes should trigger autoupdate. Skip: + * + * GENERATED / GENERATED_DIR — autoupdate writes its own outputs, + * and FSEvents delivers those writes back to us. Counting them + * would fire a follow-up autoupdate on every cycle. + * + * CMD — when a generated file changes, tup_file_mod_mtime flags + * the producing command as MODIFY so the next explicit `tup + * upd` re-runs it. That's a downstream propagation, not a + * user edit, and we'd otherwise loop on the cmd's flag set + * by our own do_scan after each autoupdate cycle. + * + * Inotify reaches the same shape via per-event filtering inside + * handle_event (TUP_NODE_GENERATED skips in the IN_CREATE/ + * IN_MODIFY branches). We filter at the flag-scan level instead. + */ + if(tent && (tent->type == TUP_NODE_GENERATED || + tent->type == TUP_NODE_GENERATED_DIR || + tent->type == TUP_NODE_CMD)) + return 0; + *(int*)arg = 1; + return 0; +} + +/* FSEvents callback - runs on the FSEvents dispatch queue thread */ +static void fsevents_callback(ConstFSEventStreamRef streamRef, + void *clientCallBackInfo, + size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]) +{ + size_t i; + char **paths = (char **)eventPaths; + + (void)streamRef; + (void)clientCallBackInfo; + (void)eventIds; + + for(i = 0; i < numEvents; i++) { + FSEventStreamEventFlags flags = eventFlags[i]; + + DEBUGP("FSEvent: '%s' flags=%08x\n", paths[i], flags); + + if(flags & (kFSEventStreamEventFlagRootChanged | + kFSEventStreamEventFlagMustScanSubDirs)) { + DEBUGP("root changed or must-rescan\n"); + } + + pthread_mutex_lock(&event_lock); + events_occurred = 1; + pthread_mutex_unlock(&event_lock); + } +} + +static int wp_callback(tupid_t newdt, const char *file, int *skip) +{ + (void)newdt; + (void)file; + (void)skip; + return 0; +} + +/* Check if there are pending modifications and trigger autoupdate or + * autoparse if configured. Returns 0 on success, -1 on error. + * + * Gated on AUTOUPDATE_PID being clear. Without that gate, calling this + * twice in quick succession (e.g. once from the main loop and again + * from the yield path before the spawned `tup autoupdate` has acquired + * the lock and cleared the create/modify flags) fires a second + * autoupdate against the same flags — and the second autoupdate then + * contends for the lock too, triggering a third yield+fire, etc. The + * AUTOUPDATE_PID config entry is set by autoupdate() and cleared by + * the `tup autoupdate` child when it exits, so checking it tells us + * whether a prior fire is still in flight. + */ +static int try_autoupdate(void) +{ + int modified = 0; + int pid; + const char *cmd; + + if(autoupdate_enabled()) + cmd = "autoupdate"; + else if(autoparse_enabled()) + cmd = "autoparse"; + else + return 0; + + if(tup_db_begin() < 0) + return -1; + if(tup_db_config_get_int(AUTOUPDATE_PID, -1, &pid) < 0) + return -1; + if(pid == -1) { + if(tup_db_select_node_by_flags(mod_cb, &modified, + TUP_FLAGS_CREATE) < 0) + return -1; + if(strcmp(cmd, "autoupdate") == 0 && + tup_db_select_node_by_flags(mod_cb, &modified, + TUP_FLAGS_MODIFY) < 0) + return -1; + } + if(tup_db_commit() < 0) + return -1; + + if(pid == -1 && modified && autoupdate(cmd) < 0) + return -1; + return 0; +} + +static int do_scan(void) +{ + struct timespan ts; + int rc; + + timespan_start(&ts); + + /* Clear variant state before re-scanning, since tup_db_scan_begin() + * will reload variants from the database. + */ + variants_free(); + if(tup_entry_clear() < 0) + return -1; + + if(tup_db_scan_begin() < 0) + return -1; + /* Enable inode-based rename detection for this scan only. The + * one-shot scanner (`tup scan` / `tup upd` without monitor) keeps + * the legacy delete+create semantics — tests like t5012 rely on + * it. Renames inside an active monitor session, by contrast, + * should match what inotify's MOVED_FROM/MOVED_TO pair handling + * delivers: identity preserved. + */ + watch_path_set_detect_rename(1); + rc = watch_path(0, ".", wp_callback); + watch_path_set_detect_rename(0); + if(rc < 0) + return -1; + if(tup_db_scan_end() < 0) + return -1; + timespan_end(&ts); + DEBUGP("Scan completed in %f seconds.\n", timespan_seconds(&ts)); + return 0; +} + +/* Wait for NOTE_FUNLOCK on the object lock file using kqueue. This fires + * the instant another process releases or closes its flock on the file. + * Returns: 0 = lock released, -1 = error + */ +static int wait_obj_unlock(void) +{ + int kq; + struct kevent ev; + int ret; + + kq = kqueue(); + if(kq < 0) { + perror("kqueue"); + return -1; + } + + EV_SET(&ev, tup_obj_lock(), EVFILT_VNODE, EV_ADD | EV_ENABLE, + NOTE_FUNLOCK, 0, NULL); + if(kevent(kq, &ev, 1, NULL, 0, NULL) < 0) { + perror("kevent register NOTE_FUNLOCK"); + close(kq); + return -1; + } + + /* Block until the lock is released */ + ret = kevent(kq, NULL, 0, &ev, 1, NULL); + close(kq); + if(ret < 0) { + if(errno == EINTR) + return 0; + perror("kevent wait NOTE_FUNLOCK"); + return -1; + } + return 0; +} + +/* Check if another tup process wants the object lock. + * + * The tri-lock protocol: + * - The monitor holds the object lock and has released the shared lock. + * - When another tup process wants to run, it first acquires the shared + * lock, then tries to flock the object lock (blocking). + * - We detect this by probing the shared lock every 100ms. If we can't + * get it, someone else has it and is about to want the object lock. + * - Yield: take tri-lock, release obj-lock, wait for NOTE_FUNLOCK on + * obj-lock (fired when the other process releases/closes it), then + * re-acquire obj-lock and release tri-lock. + * + * Returns: 0 = still online or back online, -1 = error + */ +static int check_lock_state(void) +{ + if(!locked) return 0; + + /* Probe: can we acquire the shared lock? If yes, nobody else is + * active, so release it and continue. If no, someone has it and + * we need to yield. + */ + int sh_rc = tup_try_flock(tup_sh_lock()); + if(sh_rc < 0) { + return -1; + } else if(sh_rc == 0) { + if(tup_unflock(tup_sh_lock()) < 0) { + return -1; + } else { + return 0; + } + } + + /* sh_rc == 1: Another process has the shared lock and is about to + * want the object lock. Yield using the tri-lock protocol. + */ + + DEBUGP("shared lock contention - yielding\n"); + + /* Always run do_scan before releasing the lock, regardless of + * events_occurred. FSEvents has ~100ms debounce/delivery latency, + * so a change made just before the contending process called + * `tup upd` may not have fired our callback yet by the time the + * 100ms-poll detects contention — the events_occurred flag races + * the lock probe. A stat-walk on yield always closes that window. + * Inotify reaches the same shape by draining its event queue at + * the yield (inotify.c handle of IN_OPEN on obj_wd → flush_queue). + * + * try_autoupdate() also runs so `tup flush` can observe the + * AUTOUPDATE_PID it polls for. The bomb risk (autoupdate's own + * writes triggering more autoupdate) is contained by mod_cb's + * GENERATED/CMD filter, not by suppressing autoupdate at the yield. + */ + pthread_mutex_lock(&event_lock); + events_occurred = 0; + pthread_mutex_unlock(&event_lock); + if(do_scan() < 0) + return -1; + if(try_autoupdate() < 0) + return -1; + + locked = 0; + + /* Take tri-lock (ensures we're first to get obj-lock back) */ + if(tup_flock(tup_tri_lock()) < 0) + return -1; + + /* Release obj-lock so the other process can proceed */ + if(tup_unflock(tup_obj_lock()) < 0) + return -1; + + DEBUGP("monitor off\n"); + + /* Wait for the other process to release the object lock. kqueue + * NOTE_FUNLOCK fires the instant flock is released or the fd is + * closed - no polling, no race. + */ + if(wait_obj_unlock() < 0) + return -1; + + /* Re-acquire obj-lock */ + if(tup_flock(tup_obj_lock()) < 0) + return -1; + + /* Release tri-lock so the other process can finish tup_lock_exit() + * (it waits on tri-lock before releasing the shared lock). + */ + if(tup_unflock(tup_tri_lock()) < 0) + return -1; + + /* During an update, generated nodes and ghost nodes may be removed. + * The monitor needs to invalidate those entries. We just clear out + * the cache and rebuild from the database as necessary. + */ + if(tup_entry_clear() < 0) + return -1; + + /* Reload the variants, since we may have new ones or have deleted + * old ones during the update. + */ + variants_free(); + if(tup_db_begin() < 0) + return -1; + if(variant_load() < 0) + return -1; + if(tup_db_commit() < 0) + return -1; + + locked = 1; + DEBUGP("monitor ON\n"); + + return 0; +} + +static int monitor_loop(void) +{ + int rc; + struct timespan ts; + FSEventStreamRef stream; + CFStringRef path_to_watch; + CFArrayRef paths_to_watch; + FSEventStreamContext ctx = {0, NULL, NULL, NULL, NULL}; + dispatch_queue_t queue; + + timespan_start(&ts); + + /* Initial scan */ + if(tup_db_scan_begin() < 0) + return -1; + if(watch_path(0, ".", wp_callback) < 0) + return -1; + if(tup_db_scan_end() < 0) + return -1; + + /* If we are running in autoupdate mode, we should check to see if + * any files were modified while the monitor wasn't running. If so, + * we should run an update right away. + */ + if(try_autoupdate() < 0) + return -1; + + timespan_end(&ts); + fprintf(stderr, "Initialized in %f seconds.\n", timespan_seconds(&ts)); + + /* In background mode, signal the parent that we're ready. The + * parent is blocked on kqueue waiting for NOTE_ATTRIB on the + * obj-lock file before it exits. + */ + if(backgrounding) { + if(ftruncate(tup_obj_lock(), 0) < 0) { + perror("ftruncate obj lock"); + return -1; + } + backgrounding = 0; + } + + /* Create FSEvents stream */ + path_to_watch = CFStringCreateWithCString(NULL, get_tup_top(), kCFStringEncodingUTF8); + if(!path_to_watch) { + fprintf(stderr, "tup error: Failed to create CFString for path\n"); + return -1; + } + paths_to_watch = CFArrayCreate(NULL, (const void **)&path_to_watch, 1, &kCFTypeArrayCallBacks); + if(!paths_to_watch) { + CFRelease(path_to_watch); + fprintf(stderr, "tup error: Failed to create CFArray\n"); + return -1; + } + + stream = FSEventStreamCreate(NULL, + &fsevents_callback, + &ctx, + paths_to_watch, + kFSEventStreamEventIdSinceNow, + 0.1, /* 100ms debounce */ + kFSEventStreamCreateFlagFileEvents); + + CFRelease(paths_to_watch); + CFRelease(path_to_watch); + + if(!stream) { + fprintf(stderr, "tup error: Failed to create FSEvent stream\n"); + return -1; + } + + queue = dispatch_queue_create("tup.monitor.fsevents", DISPATCH_QUEUE_SERIAL); + FSEventStreamSetDispatchQueue(stream, queue); + + if(!FSEventStreamStart(stream)) { + fprintf(stderr, "tup error: Failed to start FSEvent stream\n"); + FSEventStreamInvalidate(stream); + FSEventStreamRelease(stream); + dispatch_release(queue); + return -1; + } + + /* Main event loop: poll for FSEvents + check lock state periodically */ + do { + struct timeval tv; + int have_events; + int ret; + + /* Sleep 100ms between iterations */ + tv.tv_sec = 0; + tv.tv_usec = 100000; + ret = select(0, NULL, NULL, NULL, &tv); + if(ret < 0 && errno != EINTR) { + perror("select"); + rc = -1; + goto stop_stream; + } + + /* Check if .tup/db still exists */ + { + struct stat st; + if(stat(TUP_DIR "/db", &st) < 0 && errno == ENOENT) { + printf("tup monitor: .tup file 'db' deleted - shutting down.\n"); + rc = 0; + goto stop_stream; + } + } + + /* Check lock state (are we still the owner?) */ + if(check_lock_state() < 0) { + rc = -1; + goto stop_stream; + } + + /* Process filesystem events if we're locked (have control) */ + if(locked) { + pthread_mutex_lock(&event_lock); + have_events = events_occurred; + events_occurred = 0; + pthread_mutex_unlock(&event_lock); + + if(have_events) { + if(do_scan() < 0 || try_autoupdate() < 0) { + rc = -1; + goto stop_stream; + } + } + } + } while(!monitor_quit); + + rc = 0; + +stop_stream: + FSEventStreamStop(stream); + FSEventStreamInvalidate(stream); + FSEventStreamRelease(stream); + dispatch_release(queue); + + monitor_set_pid(-1); + return rc; +} + +static void sighandler(int sig) +{ + if(sig == SIGUSR1) { + /* No-op on macOS */ + } else if(sig == SIGHUP) { + monitor_quit = 1; + } else { + monitor_set_pid(-1); + /* TODO: gracefully close, or something? */ + exit(0); + } +} diff --git a/src/tup/monitor/inotify.c b/src/tup/monitor/inotify.c index 9666ba575..2b524ecd9 100644 --- a/src/tup/monitor/inotify.c +++ b/src/tup/monitor/inotify.c @@ -40,6 +40,7 @@ #define _ATFILE_SOURCE #include "tup/monitor.h" +#include "monitor_common.h" #include #include #include @@ -85,14 +86,11 @@ struct moved_from_event { }; LIST_HEAD(moved_from_event_head, moved_from_event); -static int monitor_set_pid(int pid); static int monitor_loop(void); static int wp_callback(tupid_t newdt, const char *file, int *skip); static int events_queued(void); static int queue_event(struct inotify_event *e); static int flush_queue(int do_autoupdate); -static int autoupdate(const char *cmd); -static void *wait_thread(void *arg); static int skip_event(struct inotify_event *e); static int eventcmp(struct inotify_event *e1, struct inotify_event *e2); static int same_event(struct inotify_event *e1, struct inotify_event *e2); @@ -115,15 +113,6 @@ static struct sigaction sigact = { }; static struct monitor_event_head event_list; static struct monitor_event *queue_last_e = NULL; -static char **update_argv; -static int update_argc; -static int autoupdate_flag = -1; -static int autoparse_flag = -1; -static pthread_mutex_t autoupdate_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t autoupdate_cond = PTHREAD_COND_INITIALIZER; -#define AUTOUPDATE_EXIT -2 -#define AUTOUPDATE_NONE -1 -static pid_t autoupdate_pid = AUTOUPDATE_NONE; static volatile sig_atomic_t dircache_debug = 0; static volatile sig_atomic_t monitor_quit = 0; static struct moved_from_event_head moved_from_list = LIST_HEAD_INITIALIZER(&moved_from_list); @@ -135,7 +124,6 @@ int monitor_supported(void) int monitor(int argc, char **argv) { - int x; int rc = 0; int foreground; pthread_t autoupdate_thread; @@ -143,47 +131,10 @@ int monitor(int argc, char **argv) /* Close down the fork process, since we don't need it. */ if(server_post_exit() < 0) return -1; - foreground = tup_option_get_flag("monitor.foreground"); + monitor_parse_args(argc, argv, &foreground); TAILQ_INIT(&event_list); - /* Arguments are cleared to "-" if they are used by the monitor. These - * args are also passed on to the autoupdate process if that feature is - * enabled, but we don't want the updater getting any args that are - * meant for the monitor. Ultimately the options may end up at - * prune_graph(), which ignores args that begin with '-'. - */ - for(x=0; x= (signed)sizeof(buf)) { - fprintf(stderr, "Buf is sized too small in monitor_set_pid\n"); - return -1; - } - if(write(fd, buf, len) < 0) { - perror("write"); - return -1; - } - if(ftruncate(fd, len) < 0) { - perror("ftruncate"); - return -1; - } - if(tup_unflock(fd) < 0) { - return -1; - } - if(close(fd) < 0) { - perror("close(fd"); - return -1; - } - return 0; -} - -int monitor_get_pid(int restarting, int *pid) -{ - struct buf b; - int fd; - - *pid = -1; - fd = openat(tup_top_fd(), MONITOR_PID_FILE, O_RDWR, 0666); - if(fd < 0) { - if(errno != ENOENT) { - perror(MONITOR_PID_FILE); - return -1; - } - /* No pid file means we don't have the monitor running, so just - * leave it at -1 and return success. - */ - return 0; - } - if(tup_flock(fd) < 0) { - return -1; - } - if(fslurp_null(fd, &b) < 0) { - goto out; - } - - if(b.len > 0) { - *pid = strtol(b.s, NULL, 0); - } - free(b.s); -out: - if(tup_unflock(fd) < 0) { - return -1; - } - if(close(fd) < 0) { - perror("close(fd"); - return -1; - } - - if(*pid > 0) { - /* Just using getpriority() to see if the monitor process is - * alive. - */ - errno = 0; - if(getpriority(PRIO_PROCESS, *pid) == -1 && errno == ESRCH) { - printf("Monitor pid %i doesn't exist anymore.\n", *pid); - if(restarting == TUP_MONITOR_RESTARTING) { - /* If we are actually restarting the monitor - * make sure we let them know that the 'pid - * doesn't exist anymore' message isn't just - * an error message. - */ - printf("Restarting the monitor.\n"); - } - monitor_set_pid(-1); - *pid = -1; - } - } - return 0; -} - -static int autoupdate_enabled(void) -{ - int autoupdate_config; - if(autoupdate_flag == 1) - return 1; - autoupdate_config = tup_option_get_flag("monitor.autoupdate"); - if(autoupdate_flag == -1 && autoupdate_config == 1) - return 1; - return 0; -} - -static int autoparse_enabled(void) -{ - int autoparse_config; - if(autoparse_flag == 1) - return 1; - autoparse_config = tup_option_get_flag("monitor.autoparse"); - if(autoparse_flag == -1 && autoparse_config == 1) - return 1; - return 0; -} - static int mod_cb(void *arg, struct tup_entry *tent) { if(tent) {} @@ -677,37 +510,6 @@ static int monitor_loop(void) return 0; } -int stop_monitor(int restarting) -{ - int pid; - - if(monitor_get_pid(restarting, &pid) < 0) { - fprintf(stderr, "tup error: Unable to get the current monitor pid in order to shut it down.\n"); - return -1; - } - if(pid < 0) { - if(restarting == TUP_MONITOR_SHUTDOWN) { - /* This case returns an error so we can tell in the - * test code if the monitor isn't actually running when - * it should be. - */ - printf("No monitor process to kill (pid < 0)\n"); - return -1; - } - return 0; - } - if(restarting == TUP_MONITOR_RESTARTING) - printf("Restarting the monitor.\n"); - else - printf("Shutting down the monitor.\n"); - if(kill(pid, SIGHUP) < 0) { - perror("kill"); - return -1; - } - - return 0; -} - static int wp_callback(tupid_t newdt, const char *file, int *skip) { int wd; @@ -855,112 +657,6 @@ static int flush_queue(int do_autoupdate) return 0; } -static int autoupdate(const char *cmd) -{ - /* This runs in a separate process (as opposed to just calling - * updater() directly) so it can properly get the lock from us (the - * monitor) and flush the queue correctly. Otherwise files touched by - * the updater will be caught by us after we return to regular event - * processing mode, which is annoying. - */ - pid_t pid = fork(); - if(pid < 0) { - perror("fork"); - return -1; - } - if(pid == 0) { - char **args; - int x; - - args = malloc((sizeof *args) * (update_argc + 4)); - if(!args) { - perror("malloc"); - exit(1); - } - args[0] = strdup("tup"); - if(!args[0]) { - perror("strdup"); - exit(1); - } - args[1] = strdup(cmd); - if(!args[1]) { - perror("strdup"); - exit(1); - } - args[2] = strdup("--no-environ-check"); - if(!args[2]) { - perror("strdup"); - exit(1); - } - for(x=0; x