Common file system utilities. More...

#include <fs.h>

Static Public Member Functions
static std::map< std::string, FileType >	scan_dir (const std::string &, std::function< bool(const std::string &, FileType)>=default_scan_filter, std::system_error *=nullptr)
	Scan a directory, and return a map of names and file types. More...

static FileType	file_type (const std::string &, std::system_error *=nullptr)
	Get the file type. More...

static FileStat	file_stat (const std::string &, std::system_error *=nullptr)
	Get the file stat. More...

static void	set_mode (const std::string &, unsigned, std::system_error *=nullptr)
	Set the file mode. More...

static void	set_times (const std::string &, uint64_t, uint64_t, std::system_error *=nullptr)
	Set the file times. More...

static void	set_ids (const std::string &, int, int, std::system_error *=nullptr)
	Set the user id and/or group id. More...

static void	set_size (const std::string &, off_t, std::system_error *=nullptr)
	Set the file size. More...

static std::map< off_t, off_t >	sparse_map (const std::string &, std::system_error *=nullptr)
	Map of file sparseness. More...

static void	remove (const std::string &, std::system_error *=nullptr)
	Remove the file or directory. More...

static void	rename (const std::string &, const std::string &, std::system_error *=nullptr)
	Rename the file or directory. More...

static void	remove_all (const std::string &fn, std::system_error *err=nullptr)
	Recursively removes the file or directory. More...

static void	create_dir (const std::string &, std::system_error *=nullptr)
	Create a directory. More...

static void	create_reg (const std::string &, std::system_error *=nullptr)
	Create a regular file. More...

static std::string	create_tmp_reg (const std::string &, std::system_error *=nullptr)
	Create a temporary regular file. More...

static void	create_symlink (const std::string &, const std::string &, std::system_error *=nullptr)
	Create a symbolic link. More...

static void	create_link (const std::string &, const std::string &, std::system_error *=nullptr)
	Create a hard link. More...

static void	create_fifo (const std::string &, std::system_error *=nullptr)
	Create a named pipe (FIFO). More...

static void	change_dir (const std::string &, std::system_error *=nullptr)
	Change working directory. More...

static std::string	get_current_dir (std::system_error *=nullptr)
	Get working directory. More...

static std::string	read_symlink (const std::string &, std::system_error *=nullptr)
	Read the location of a symbolic link target. More...

static std::string	norm_path (const std::string &, const std::string &) noexcept
	Normalize a path with base directory. More...

static std::string	norm_path (const std::string &path) noexcept
	Normalize a path. More...

Detailed Description

Common file system utilities.

Utility to manipulate files and the filesystem.

Examples: crypto/unittest/secvec.cc, util/unittest/fs.cc, and util/unittest/logger.cc.

Definition at line 102 of file fs.h.

Member Function Documentation

◆ change_dir()

void Filesystem::change_dir	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Change working directory.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc, and util/unittest/logger.cc.

Definition at line 440 of file fs.cc.

◆ create_dir()

void Filesystem::create_dir	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Create a directory.

On error, throws an exception or sets err if provided.

Parameters

name	File name.
err	Exception override.

The directory will be created if necessary with mode 0700.

Examples: util/unittest/fs.cc, and util/unittest/logger.cc.

Definition at line 284 of file fs.cc.

◆ create_fifo()

void Filesystem::create_fifo	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Create a named pipe (FIFO).

On error, throws an exception or sets err if provided.

The file will be created if necessary with mode 0600.

Examples: util/unittest/fs.cc.

Definition at line 386 of file fs.cc.

◆ create_link()

void Filesystem::create_link	(	const std::string &	orig_name,
		const std::string &	new_name,
		std::system_error *	err = `nullptr`
	)

static

Create a hard link.

Results in a second inode that points to the original file.

Parameters

orig_name	Original file name.
new_name	New file name.
err	Exception override.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 372 of file fs.cc.

◆ create_reg()

void Filesystem::create_reg	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Create a regular file.

Parameters

name	File name.
err	Exception override.

The file will be created if necessary with mode 0600.

If the file exists, and cannot be opened for writing, throws an error.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 298 of file fs.cc.

◆ create_symlink()

void Filesystem::create_symlink	(	const std::string &	target,
		const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Create a symbolic link.

Parameters

target	Target of the link.
name	Path name of the link file.
err	Exception override.

E.g. create_symlink("../reg", "dir/link") creates a symlink: dir/link -> ../reg

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 339 of file fs.cc.

◆ create_tmp_reg()

std::string Filesystem::create_tmp_reg	(	const std::string &	prefix,
		std::system_error *	err = `nullptr`
	)

static

Create a temporary regular file.

Return the filename, which will be prefix plus a random string.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 314 of file fs.cc.

◆ file_stat()

FileStat Filesystem::file_stat	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Get the file stat.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 400 of file fs.cc.

◆ file_type()

FileType Filesystem::file_type	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Get the file type.

On error, throws an exception or sets err if provided.

Definition at line 186 of file fs.cc.

◆ get_current_dir()

std::string Filesystem::get_current_dir ( std::system_error * err = nullptr )

static

Get working directory.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc, and util/unittest/logger.cc.

Definition at line 454 of file fs.cc.

◆ norm_path() [1/2]

std::string Filesystem::norm_path	(	const std::string &	base_dir,
		const std::string &	path
	)

staticnoexcept

Normalize a path with base directory.

If path is relative, will append to the base dir, and return a normalized version.

If path is absolute, ignores base_dir, and returns a normalized version of path.

Normalized version removes all consecutive slashes, and unneccesary . and .. directories.

Returned paths starting with:

"/" is an absolute path
"/.." is an invalid absolute path
"." is a relative path starting in base directory
".." is never returned ("./.." may be returned)
any other is relative

An input of "" paths will result in output "."

Conserves trailing slash, if any.

Parameters

base_dir	Base directory
path	Path

Examples: util/unittest/fs.cc.

Definition at line 671 of file fs.cc.

◆ norm_path() [2/2]

static std::string scc::util::Filesystem::norm_path ( const std::string & path )

inlinestaticnoexcept

Normalize a path.

Parameters

path Path

Definition at line 340 of file fs.h.

◆ read_symlink()

std::string Filesystem::read_symlink	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Read the location of a symbolic link target.

Parameters

name	The symbolic link name
err	Exception override.

Returns: target_name File name of the symbolic link target

Example if "dir/link" -> "dir/regular", read_symlink(dir/link) will return "dir/regular"

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 353 of file fs.cc.

◆ remove()

void Filesystem::remove	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Remove the file or directory.

On error, throws an exception or sets err if provided.

Examples: crypto/unittest/secvec.cc, net/unittest/unix.cc, and util/unittest/fs.cc.

Definition at line 233 of file fs.cc.

◆ remove_all()

static void scc::util::Filesystem::remove_all	(	const std::string &	fn,
		std::system_error *	err = `nullptr`
	)

inlinestatic

Recursively removes the file or directory.

On error, throws an exception or sets err if provided.

Definition at line 226 of file fs.h.

◆ rename()

void Filesystem::rename	(	const std::string &	old_fn,
		const std::string &	new_fn,
		std::system_error *	err = `nullptr`
	)

static

Rename the file or directory.

On error, throws an exception or sets err if provided.

Parameters

old_fn	Old filename.
new_fn	New Filename.

Definition at line 270 of file fs.cc.

◆ scan_dir()

std::map< std::string, FileType > Filesystem::scan_dir	(	const std::string &	dirname,
		std::function< bool(const std::string &, FileType)>	filter = `default_scan_filter`,
		std::system_error *	err = `nullptr`
	)

static

Scan a directory, and return a map of names and file types.

Does not scan recursively.

Parameters

dirname	The directory name.
filter	Scan filter function which is called with parameters (name, FileType), and returns true if the file should be included.
err	If set, on error will set this value and the exception will not be thrown.

Returns: map<name, FileType> Map of files found, empty if no files were found.

Example in scclib/util/unittest/fs.cc

static bool reg_filt(const string& s, FileType t)
{
    if (s == "reg" && t == FileType::reg)
        return true;
    return false;
}
 
 
TEST_F(FsTest, filter)
{
    create_files();
    auto files = fs::scan_dir("sandbox", reg_filt);
    ASSERT_EQ(files.size(), 1);
    ASSERT_EQ(files.begin()->first, "reg");
    ASSERT_EQ(files.begin()->second, FileType::reg);
}
 

Examples: util/unittest/fs.cc.

Definition at line 128 of file fs.cc.

◆ set_ids()

void Filesystem::set_ids	(	const std::string &	name,
		int	uid,
		int	gid,
		std::system_error *	err = `nullptr`
	)

static

Set the user id and/or group id.

If either value is < 0, it will not be set.

Parameters

name	File name.
uid	User id.
gid	Group id.
err	Exception override.

Definition at line 487 of file fs.cc.

◆ set_mode()

void Filesystem::set_mode	(	const std::string &	name,
		unsigned	mode,
		std::system_error *	err = `nullptr`
	)

static

Set the file mode.

Parameters

name	File name.
mode	File mode, e.g. 0664.
err	Exception override.

File mode can be specified in octal format:

0400 read by owner
0200 write by owner
0100 execute/search directory by owner
0040 read by group
0020 write by group
0010 execute/search directory by group
0004 read by group
0002 write by group
0001 execute/search directory by group

Calling this api on a symbolic link will result in an error.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 472 of file fs.cc.

◆ set_size()

void Filesystem::set_size	(	const std::string &	name,
		off_t	size,
		std::system_error *	err = `nullptr`
	)

static

Set the file size.

Parameters

name	File name.
size	New file size.
err	Exception override.

If size is less than current size, file is truncated.

If size is greater than current size, a sparse file is created.

Example in scclib/util/unittest/fs.cc

TEST(FsExampleTest, sparse_and_trunc)
{
    system_error err;
    fs::remove_all("sandbox", &err);
    fs::create_dir("sandbox");
    auto curdir = fs::get_current_dir();
    cout << "curdir: " << curdir << endl;
 
    fs::change_dir("sandbox");
 
    string s("this is a test of the emergency sizing system\n");
    cout << "sparse test with string len=" << s.size() << endl;
 
    string fn("sparse_file");
 
    auto write_sparse = [&fn, &s](off_t loc)
    {
        // use clib to write data into the middle of the file
        // if we use fstream, the system may truncate the file
 
        // use system wrapper FileDesc to ensure the file is always closed
        scc::util::FileDesc fd(scc::util::safe_open_throw(fn.c_str(), O_WRONLY));
        
        lseek(fd, loc, SEEK_SET);
        safe_write_throw(fd, &s[0], s.size());
        
        cout << "wrote " << s.size() << " bytes at loc " << loc << endl;
    };
 
    fs::create_reg(fn);
    fs::set_size(fn, 16384);            // 16 K file, with a 0 at the end
    
    auto st = fs::file_stat(fn);
    cout << "12288 hole at start of file:\n" << st << endl;
 
    ASSERT_EQ(st.size, 16384);
    ASSERT_EQ(st.alloc_size, 4096);     // the last block has data
 
    auto sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
    
    std::map<int64_t,int64_t> ver;
    
    ver[0] = 12287;                     // 12288 sized hole at 0 (beginning)
    ASSERT_EQ(sm, ver);
 
    write_sparse(1024);                 // write the string at 1024, causing first block to be written
    
    st = fs::file_stat(fn);
    cout << "8192 hole in middle of file\n" << st << endl;
 
    ASSERT_EQ(st.alloc_size, 8192);
 
    sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
 
    ver.clear();
    ver[4096] = 12287;                  // 8192 sized hole at 4096 (middle)
    ASSERT_EQ(sm, ver);
 
    fs::set_size(fn, 12288);
 
    st = fs::file_stat(fn);
    cout << "8192 hole at end of file\n" << st << endl;
 
    ASSERT_EQ(st.alloc_size, 4096);
 
    sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
 
    ver.clear();
    ver[4096] = 12287;                  // 8192 sized hole at 4096 (end)
    ASSERT_EQ(sm, ver);
 
    ifstream f(fn);
    f.seekg(1024);
    string val;
    val.resize(s.size(), '\x01');
    f.read(&val[0], val.size());
    ASSERT_EQ(val, s);
 
    fs::change_dir(curdir);
    fs::remove_all("sandbox");
}

Examples: util/unittest/fs.cc.

Definition at line 610 of file fs.cc.

◆ set_times()

void Filesystem::set_times	(	const std::string &	name,
		uint64_t	access_time,
		uint64_t	mod_time,
		std::system_error *	err = `nullptr`
	)

static

Set the file times.

Times are in nanoseconds since epoch (divide by 1e9 to get time_t)

Parameters

name	File name.
access_time	File access time.
mod_time	File modification time.
err	Exception override.

On error, throws an exception or sets err if provided.

Examples: util/unittest/fs.cc.

Definition at line 504 of file fs.cc.

◆ sparse_map()

std::map< off_t, off_t > Filesystem::sparse_map	(	const std::string &	name,
		std::system_error *	err = `nullptr`
	)

static

Map of file sparseness.

Parameters

name	File name.
err	Exception override.

Returns: Map of start and end offsets of holes (unwritten zero regions) in file.

A sparse file will contain data only in blocks which have been written. When data is read from an unwritten "hole", 0 is returned.

For example, a 16 K sparse file with a single 0 written at the front and end will have map:

<4096, 12287>   - block from 4096 to 12287 is hole

This file reads all 0, but takes up 8192 bytes on disk instead of 16384.

Example in scclib/util/unittest/fs.cc

TEST(FsExampleTest, sparse_and_trunc)
{
    system_error err;
    fs::remove_all("sandbox", &err);
    fs::create_dir("sandbox");
    auto curdir = fs::get_current_dir();
    cout << "curdir: " << curdir << endl;
 
    fs::change_dir("sandbox");
 
    string s("this is a test of the emergency sizing system\n");
    cout << "sparse test with string len=" << s.size() << endl;
 
    string fn("sparse_file");
 
    auto write_sparse = [&fn, &s](off_t loc)
    {
        // use clib to write data into the middle of the file
        // if we use fstream, the system may truncate the file
 
        // use system wrapper FileDesc to ensure the file is always closed
        scc::util::FileDesc fd(scc::util::safe_open_throw(fn.c_str(), O_WRONLY));
        
        lseek(fd, loc, SEEK_SET);
        safe_write_throw(fd, &s[0], s.size());
        
        cout << "wrote " << s.size() << " bytes at loc " << loc << endl;
    };
 
    fs::create_reg(fn);
    fs::set_size(fn, 16384);            // 16 K file, with a 0 at the end
    
    auto st = fs::file_stat(fn);
    cout << "12288 hole at start of file:\n" << st << endl;
 
    ASSERT_EQ(st.size, 16384);
    ASSERT_EQ(st.alloc_size, 4096);     // the last block has data
 
    auto sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
    
    std::map<int64_t,int64_t> ver;
    
    ver[0] = 12287;                     // 12288 sized hole at 0 (beginning)
    ASSERT_EQ(sm, ver);
 
    write_sparse(1024);                 // write the string at 1024, causing first block to be written
    
    st = fs::file_stat(fn);
    cout << "8192 hole in middle of file\n" << st << endl;
 
    ASSERT_EQ(st.alloc_size, 8192);
 
    sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
 
    ver.clear();
    ver[4096] = 12287;                  // 8192 sized hole at 4096 (middle)
    ASSERT_EQ(sm, ver);
 
    fs::set_size(fn, 12288);
 
    st = fs::file_stat(fn);
    cout << "8192 hole at end of file\n" << st << endl;
 
    ASSERT_EQ(st.alloc_size, 4096);
 
    sm = fs::sparse_map(fn);
    cout << setw(6) << "start" << setw(6) << "data" << endl;
    for (auto& x : sm)
    {
        cout << setw(6) << x.first << setw(6) << x.second << endl;
    }
 
    ver.clear();
    ver[4096] = 12287;                  // 8192 sized hole at 4096 (end)
    ASSERT_EQ(sm, ver);
 
    ifstream f(fn);
    f.seekg(1024);
    string val;
    val.resize(s.size(), '\x01');
    f.read(&val[0], val.size());
    ASSERT_EQ(val, s);
 
    fs::change_dir(curdir);
    fs::remove_all("sandbox");
}

Examples: util/unittest/fs.cc.

Definition at line 525 of file fs.cc.

The documentation for this class was generated from the following files:

util/pub/util/fs.h
util/fs.cc

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ change_dir()

◆ create_dir()

◆ create_fifo()

◆ create_link()

◆ create_reg()

◆ create_symlink()

◆ create_tmp_reg()

◆ file_stat()

◆ file_type()

◆ get_current_dir()

◆ norm_path() [1/2]

◆ norm_path() [2/2]

◆ read_symlink()

◆ remove()

◆ remove_all()

◆ rename()

◆ scan_dir()

◆ set_ids()

◆ set_mode()

◆ set_size()

◆ set_times()

◆ sparse_map()