From: Rob Browning Date: Thu, 10 Feb 2011 05:01:45 +0000 (-0600) Subject: Merge remote branch 'origin/master' into meta X-Git-Url: https://git.michaelhowe.org/gitweb/?a=commitdiff_plain;h=6f6a1111ea20316499ad748a1cb118ec5d5c117f;p=packages%2Fb%2Fbup.git Merge remote branch 'origin/master' into meta Conflicts: lib/bup/_helpers.c lib/bup/helpers.py lib/bup/t/thelpers.py t/test.sh --- 6f6a1111ea20316499ad748a1cb118ec5d5c117f diff --cc cmd/meta-cmd.py index f1f5a7b,0000000..4f6e013 mode 100755,000000..100755 --- a/cmd/meta-cmd.py +++ b/cmd/meta-cmd.py @@@ -1,148 -1,0 +1,148 @@@ +#!/usr/bin/env python + +# Copyright (C) 2010 Rob Browning +# +# This code is covered under the terms of the GNU Library General +# Public License as described in the bup LICENSE file. + +# TODO: Add tar-like -C option. +# TODO: Add tar-like -v support to --list. + +import sys +from bup import metadata +from bup import options +from bup.helpers import handle_ctrl_c, log, saved_errors + +optspec = """ +bup meta --create [OPTION ...] +bup meta --extract [OPTION ...] +bup meta --start-extract [OPTION ...] +bup meta --finish-extract [OPTION ...] +-- +c,create write metadata for PATHs to stdout (or --file) +t,list display metadata +x,extract perform --start-extract followed by --finish-extract +start-extract build tree matching metadata provided on standard input (or --file) +finish-extract finish applying standard input (or --file) metadata to filesystem +f,file= specify source or destination file +R,recurse recurse into subdirectories +xdev,one-file-system don't cross filesystem boundaries +numeric-ids apply numeric IDs (user, group, etc.), not names, during restore +symlinks handle symbolic links (default is true) +paths include paths in metadata (default is true) +v,verbose increase log output (can be used more than once) +q,quiet don't show progress meter +""" + +action = None +target_filename = '' +should_recurse = False +restore_numeric_ids = False +include_paths = True +handle_symlinks = True +xdev = False + +handle_ctrl_c() + - o = options.Options('bup meta', optspec) ++o = options.Options(optspec) +(opt, flags, remainder) = o.parse(sys.argv[1:]) + +for flag, value in flags: + if flag == '--create' or flag == '-c': + action = 'create' + elif flag == '--list' or flag == '-t': + action = 'list' + elif flag == '--extract' or flag == '-x': + action = 'extract' + elif flag == '--start-extract': + action = 'start-extract' + elif flag == '--finish-extract': + action = 'finish-extract' + elif flag == '--file' or flag == '-f': + target_filename = value + elif flag == '--recurse' or flag == '-R': + should_recurse = True + elif flag == '--no-recurse': + should_recurse = False + elif flag in frozenset(['--xdev', '--one-file-system']): + xdev = True + elif flag in frozenset(['--no-xdev', '--no-one-file-system']): + xdev = False + elif flag == '--numeric-ids': + restore_numeric_ids = True + elif flag == '--no-numeric-ids': + restore_numeric_ids = False + elif flag == '--paths': + include_paths = True + elif flag == '--no-paths': + include_paths = False + elif flag == '--symlinks': + handle_symlinks = True + elif flag == '--no-symlinks': + handle_symlinks = False + elif flag == '--verbose' or flag == '-v': + metadata.verbose += 1 + elif flag == '--quiet' or flag == '-q': + metadata.verbose = 0 + +if not action: + o.fatal("no action specified") + +if action == 'create': + if len(remainder) < 1: + o.fatal("no paths specified for create") + if target_filename != '-': + output_file = open(target_filename, 'w') + else: + output_file = sys.stdout + metadata.save_tree(output_file, + remainder, + recurse=should_recurse, + write_paths=include_paths, + save_symlinks=handle_symlinks, + xdev=xdev) + +elif action == 'list': + if len(remainder) > 0: + o.fatal("cannot specify paths for --list") + if target_filename != '-': + src = open(target_filename, 'r') + else: + src = sys.stdin + metadata.display_archive(src) + +elif action == 'start-extract': + if len(remainder) > 0: + o.fatal("cannot specify paths for --start-extract") + if target_filename != '-': + src = open(target_filename, 'r') + else: + src = sys.stdin + metadata.start_extract(src, create_symlinks=handle_symlinks) + +elif action == 'finish-extract': + if len(remainder) > 0: + o.fatal("cannot specify paths for --finish-extract") + if target_filename != '-': + src = open(target_filename, 'r') + else: + src = sys.stdin + num_ids = restore_numeric_ids + metadata.finish_extract(src, restore_numeric_ids=num_ids) + +elif action == 'extract': + if len(remainder) > 0: + o.fatal("cannot specify paths for --extract") + if target_filename != '-': + src = open(target_filename, 'r') + else: + src = sys.stdin + metadata.extract(src, + restore_numeric_ids=restore_numeric_ids, + create_symlinks=handle_symlinks) + +if saved_errors: + log('WARNING: %d errors encountered.\n' % len(saved_errors)) + sys.exit(1) +else: + sys.exit(0) diff --cc cmd/xstat-cmd.py index b8ee4e8,0000000..6d60596 mode 100755,000000..100755 --- a/cmd/xstat-cmd.py +++ b/cmd/xstat-cmd.py @@@ -1,132 -1,0 +1,132 @@@ +#!/usr/bin/env python + +# Copyright (C) 2010 Rob Browning +# +# This code is covered under the terms of the GNU Library General +# Public License as described in the bup LICENSE file. + +import errno +import posix1e +import stat +import sys +from bup import metadata +from bup import options +from bup import xstat +from bup.helpers import handle_ctrl_c, saved_errors, add_error, log + + +def fstimestr(fstime): + (s, ns) = fstime.secs_nsecs() + if ns == 0: + return '%d' % s + else: + return '%d.%09d' % (s, ns) + + +optspec = """ +bup pathinfo [OPTION ...] +-- +v,verbose increase log output (can be used more than once) +q,quiet don't show progress meter +exclude-fields= exclude comma-separated fields +include-fields= include comma-separated fields (definitive if first) +""" + +target_filename = '' +all_fields = frozenset(['path', + 'mode', + 'link-target', + 'rdev', + 'uid', + 'gid', + 'owner', + 'group', + 'atime', + 'mtime', + 'ctime', + 'linux-attr', + 'linux-xattr', + 'posix1e-acl']) +active_fields = all_fields + +handle_ctrl_c() + - o = options.Options('bup pathinfo', optspec) ++o = options.Options(optspec) +(opt, flags, remainder) = o.parse(sys.argv[1:]) + +treat_include_fields_as_definitive = True +for flag, value in flags: + if flag == '--verbose' or flag == '-v': + metadata.verbose += 1 + elif flag == '--quiet' or flag == '-q': + metadata.verbose = 0 + elif flag == '--exclude-fields': + exclude_fields = frozenset(value.split(',')) + for f in exclude_fields: + if not f in all_fields: + o.fatal(f + ' is not a valid field name') + active_fields = active_fields - exclude_fields + treat_include_fields_as_definitive = False + elif flag == '--include-fields': + include_fields = frozenset(value.split(',')) + for f in include_fields: + if not f in all_fields: + o.fatal(f + ' is not a valid field name') + if treat_include_fields_as_definitive: + active_fields = include_fields + treat_include_fields_as_definitive = False + else: + active_fields = active_fields | include_fields + +for path in remainder: + try: + m = metadata.from_path(path, archive_path = path) + except IOError, e: + if e.errno == errno.ENOENT: + add_error(e) + continue + else: + raise + if 'path' in active_fields: + print 'path:', m.path + if 'mode' in active_fields: + print 'mode:', oct(m.mode) + if 'link-target' in active_fields and stat.S_ISLNK(m.mode): + print 'link-target:', m.symlink_target + if 'rdev' in active_fields: + print 'rdev:', m.rdev + if 'uid' in active_fields: + print 'uid:', m.uid + if 'gid' in active_fields: + print 'gid:', m.gid + if 'owner' in active_fields: + print 'owner:', m.owner + if 'group' in active_fields: + print 'group:', m.group + if 'atime' in active_fields: + print 'atime: ' + fstimestr(m.atime) + if 'mtime' in active_fields: + print 'mtime: ' + fstimestr(m.mtime) + if 'ctime' in active_fields: + print 'ctime: ' + fstimestr(m.ctime) + if 'linux-attr' in active_fields and m.linux_attr: + print 'linux-attr:', hex(m.linux_attr) + if 'linux-xattr' in active_fields and m.linux_xattr: + for name, value in m.linux_xattr: + print 'linux-xattr: %s -> %s' % (name, repr(value)) + if 'posix1e-acl' in active_fields and m.posix1e_acl: + flags = posix1e.TEXT_ABBREVIATE + if stat.S_ISDIR(m.mode): + acl = m.posix1e_acl[0] + default_acl = m.posix1e_acl[2] + print acl.to_any_text('posix1e-acl: ', '\n', flags) + print acl.to_any_text('posix1e-acl-default: ', '\n', flags) + else: + acl = m.posix1e_acl[0] + print acl.to_any_text('posix1e-acl: ', '\n', flags) + +if saved_errors: + log('WARNING: %d errors encountered.\n' % len(saved_errors)) + sys.exit(1) +else: + sys.exit(0) diff --cc lib/bup/_helpers.c index d5de937,0af9411..dbe64a7 --- a/lib/bup/_helpers.c +++ b/lib/bup/_helpers.c @@@ -3,18 -1,14 +3,22 @@@ #include "bupsplit.h" #include #include -#include +#include #include #include +#include + #include + #include + #include +#ifdef linux +#include +#include +#include +#include +#endif + + static int istty = 0; static PyObject *selftest(PyObject *self, PyObject *args) { @@@ -204,247 -492,7 +502,247 @@@ static PyObject *fadvise_done(PyObject } +#ifdef linux +static PyObject *bup_get_linux_file_attr(PyObject *self, PyObject *args) +{ + int rc; + unsigned long attr; + char *path; + int fd; + + if (!PyArg_ParseTuple(args, "s", &path)) + return NULL; + + fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW); + if (fd == -1) + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path); + + attr = 0; + rc = ioctl(fd, FS_IOC_GETFLAGS, &attr); + if (rc == -1) + { + close(fd); + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path); + } + + close(fd); + return Py_BuildValue("k", attr); +} + + +static PyObject *bup_set_linux_file_attr(PyObject *self, PyObject *args) +{ + int rc; + unsigned long attr; + char *path; + int fd; + + if (!PyArg_ParseTuple(args, "sk", &path, &attr)) + return NULL; + + fd = open(path, O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_NOFOLLOW); + if(fd == -1) + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path); + + rc = ioctl(fd, FS_IOC_SETFLAGS, &attr); + if (rc == -1) + { + close(fd); + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path); + } + + close(fd); + Py_RETURN_TRUE; +} +#endif /* def linux */ + + +#if defined(_ATFILE_SOURCE) \ + || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L +#define HAVE_BUP_UTIMENSAT 1 + +static PyObject *bup_utimensat(PyObject *self, PyObject *args) +{ + int rc, dirfd, flags; + char *path; + long access, access_ns, modification, modification_ns; + struct timespec ts[2]; + + if (!PyArg_ParseTuple(args, "is((ll)(ll))i", + &dirfd, + &path, + &access, &access_ns, + &modification, &modification_ns, + &flags)) + return NULL; + + if (isnan(access)) + { + PyErr_SetString(PyExc_ValueError, "access time is NaN"); + return NULL; + } + else if (isinf(access)) + { + PyErr_SetString(PyExc_ValueError, "access time is infinite"); + return NULL; + } + else if (isnan(modification)) + { + PyErr_SetString(PyExc_ValueError, "modification time is NaN"); + return NULL; + } + else if (isinf(modification)) + { + PyErr_SetString(PyExc_ValueError, "modification time is infinite"); + return NULL; + } + + if (isnan(access_ns)) + { + PyErr_SetString(PyExc_ValueError, "access time ns is NaN"); + return NULL; + } + else if (isinf(access_ns)) + { + PyErr_SetString(PyExc_ValueError, "access time ns is infinite"); + return NULL; + } + else if (isnan(modification_ns)) + { + PyErr_SetString(PyExc_ValueError, "modification time ns is NaN"); + return NULL; + } + else if (isinf(modification_ns)) + { + PyErr_SetString(PyExc_ValueError, "modification time ns is infinite"); + return NULL; + } + + ts[0].tv_sec = access; + ts[0].tv_nsec = access_ns; + ts[1].tv_sec = modification; + ts[1].tv_nsec = modification_ns; + + rc = utimensat(dirfd, path, ts, flags); + if (rc != 0) + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, path); + + Py_RETURN_TRUE; +} + +#endif /* defined(_ATFILE_SOURCE) + || _XOPEN_SOURCE >= 700 || _POSIX_C_SOURCE >= 200809L */ + + +#ifdef linux /* and likely others */ + +#define HAVE_BUP_STAT 1 +static PyObject *bup_stat(PyObject *self, PyObject *args) +{ + int rc; + char *filename; + + if (!PyArg_ParseTuple(args, "s", &filename)) + return NULL; + + struct stat st; + rc = stat(filename, &st); + if (rc != 0) + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + + return Py_BuildValue("kkkkkkkk" + "(ll)" + "(ll)" + "(ll)", + (unsigned long) st.st_mode, + (unsigned long) st.st_ino, + (unsigned long) st.st_dev, + (unsigned long) st.st_nlink, + (unsigned long) st.st_uid, + (unsigned long) st.st_gid, + (unsigned long) st.st_rdev, + (unsigned long) st.st_size, + (long) st.st_atime, + (long) st.st_atim.tv_nsec, + (long) st.st_mtime, + (long) st.st_mtim.tv_nsec, + (long) st.st_ctime, + (long) st.st_ctim.tv_nsec); +} + + +#define HAVE_BUP_LSTAT 1 +static PyObject *bup_lstat(PyObject *self, PyObject *args) +{ + int rc; + char *filename; + + if (!PyArg_ParseTuple(args, "s", &filename)) + return NULL; + + struct stat st; + rc = lstat(filename, &st); + if (rc != 0) + return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + + return Py_BuildValue("kkkkkkkk" + "(ll)" + "(ll)" + "(ll)", + (unsigned long) st.st_mode, + (unsigned long) st.st_ino, + (unsigned long) st.st_dev, + (unsigned long) st.st_nlink, + (unsigned long) st.st_uid, + (unsigned long) st.st_gid, + (unsigned long) st.st_rdev, + (unsigned long) st.st_size, + (long) st.st_atime, + (long) st.st_atim.tv_nsec, + (long) st.st_mtime, + (long) st.st_mtim.tv_nsec, + (long) st.st_ctime, + (long) st.st_ctim.tv_nsec); +} + + +#define HAVE_BUP_FSTAT 1 +static PyObject *bup_fstat(PyObject *self, PyObject *args) +{ + int rc, fd; + + if (!PyArg_ParseTuple(args, "i", &fd)) + return NULL; + + struct stat st; + rc = fstat(fd, &st); + if (rc != 0) + return PyErr_SetFromErrno(PyExc_IOError); + + return Py_BuildValue("kkkkkkkk" + "(ll)" + "(ll)" + "(ll)", + (unsigned long) st.st_mode, + (unsigned long) st.st_ino, + (unsigned long) st.st_dev, + (unsigned long) st.st_nlink, + (unsigned long) st.st_uid, + (unsigned long) st.st_gid, + (unsigned long) st.st_rdev, + (unsigned long) st.st_size, + (long) st.st_atime, + (long) st.st_atim.tv_nsec, + (long) st.st_mtime, + (long) st.st_mtim.tv_nsec, + (long) st.st_ctime, + (long) st.st_ctim.tv_nsec); +} + +#endif /* def linux */ + + - static PyMethodDef helper_methods[] = { + static PyMethodDef faster_methods[] = { { "selftest", selftest, METH_VARARGS, "Check that the rolling checksum rolls correctly (for unit tests)." }, { "blobbits", blobbits, METH_VARARGS, @@@ -488,22 -522,8 +794,23 @@@ { NULL, NULL, 0, NULL }, // sentinel }; + PyMODINIT_FUNC init_helpers(void) { - PyObject *m = Py_InitModule("_helpers", helper_methods); - Py_InitModule("_helpers", faster_methods); ++ PyObject *m = Py_InitModule("_helpers", faster_methods); + if (m == NULL) + return; +#ifdef HAVE_BUP_UTIMENSAT + PyModule_AddObject(m, "AT_FDCWD", Py_BuildValue("i", AT_FDCWD)); + PyModule_AddObject(m, "AT_SYMLINK_NOFOLLOW", + Py_BuildValue("i", AT_SYMLINK_NOFOLLOW)); +#endif +#ifdef HAVE_BUP_STAT + Py_INCREF(Py_True); + PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_True); +#else + Py_INCREF(Py_False); + PyModule_AddObject(m, "_have_ns_fs_timestamps", Py_False); +#endif + istty = isatty(2) || getenv("BUP_FORCE_TTY"); } diff --cc lib/bup/drecurse.py index 129679a,4196dec..2dbe50c --- a/lib/bup/drecurse.py +++ b/lib/bup/drecurse.py @@@ -1,6 -1,5 +1,6 @@@ - import stat + import stat, os from bup.helpers import * +import bup.xstat as xstat try: O_LARGEFILE = os.O_LARGEFILE diff --cc lib/bup/helpers.py index 2e9d6f1,da27edb..566343d --- a/lib/bup/helpers.py +++ b/lib/bup/helpers.py @@@ -1,7 -1,8 +1,9 @@@ """Helper functions and classes for bup.""" - import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re + + import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re, struct + import heapq, operator from bup import _version +import bup._helpers as _helpers # This function should really be in helpers, not in bup.options. But we # want options.py to be standalone so people can include it in other projects. diff --cc lib/bup/t/thelpers.py index 18f5e89,89cccda..31ecbb9 --- a/lib/bup/t/thelpers.py +++ b/lib/bup/t/thelpers.py @@@ -1,6 -1,4 +1,6 @@@ - import os, math ++import math + import os +import bup._helpers as _helpers - from bup.helpers import * from wvtest import * @@@ -14,10 -12,64 +14,71 @@@ def test_parse_num() WVPASSEQ(pn('1e+9 k'), 1000000000 * 1024) WVPASSEQ(pn('-3e-3mb'), int(-0.003 * 1024 * 1024)) - +@wvtest +def test_detect_fakeroot(): + if os.getenv('FAKEROOTKEY'): + WVPASS(detect_fakeroot()) + else: + WVPASS(not detect_fakeroot()) ++ + @wvtest + def test_strip_path(): + prefix = "/var/backup/daily.0/localhost" + empty_prefix = "" + non_matching_prefix = "/home" + path = "/var/backup/daily.0/localhost/etc/" + + WVPASSEQ(strip_path(prefix, path), '/etc') + WVPASSEQ(strip_path(empty_prefix, path), path) + WVPASSEQ(strip_path(non_matching_prefix, path), path) + WVEXCEPT(Exception, strip_path, None, path) + + @wvtest + def test_strip_base_path(): + path = "/var/backup/daily.0/localhost/etc/" + base_paths = ["/var", "/var/backup", "/var/backup/daily.0/localhost"] + WVPASSEQ(strip_base_path(path, base_paths), '/etc') + + @wvtest + def test_strip_symlinked_base_path(): + tmpdir = os.path.join(os.getcwd(),"test_strip_symlinked_base_path.tmp") + symlink_src = os.path.join(tmpdir, "private", "var") + symlink_dst = os.path.join(tmpdir, "var") + path = os.path.join(symlink_dst, "a") + + os.mkdir(tmpdir) + os.mkdir(os.path.join(tmpdir, "private")) + os.mkdir(symlink_src) + os.symlink(symlink_src, symlink_dst) + + result = strip_base_path(path, [symlink_dst]) + + os.remove(symlink_dst) + os.rmdir(symlink_src) + os.rmdir(os.path.join(tmpdir, "private")) + os.rmdir(tmpdir) + + WVPASSEQ(result, "/a") + + @wvtest + def test_graft_path(): + middle_matching_old_path = "/user" + non_matching_old_path = "/usr" + matching_old_path = "/home" + matching_full_path = "/home/user" + new_path = "/opt" + + all_graft_points = [(middle_matching_old_path, new_path), + (non_matching_old_path, new_path), + (matching_old_path, new_path)] + + path = "/home/user/" + + WVPASSEQ(graft_path([(middle_matching_old_path, new_path)], path), + "/home/user") + WVPASSEQ(graft_path([(non_matching_old_path, new_path)], path), + "/home/user") + WVPASSEQ(graft_path([(matching_old_path, new_path)], path), "/opt/user") + WVPASSEQ(graft_path(all_graft_points, path), "/opt/user") + WVPASSEQ(graft_path([(matching_full_path, new_path)], path), + "/opt")