diff --git a/dist/dattobd.spec b/dist/dattobd.spec index fa388e38..a2192fd5 100644 --- a/dist/dattobd.spec +++ b/dist/dattobd.spec @@ -117,7 +117,7 @@ Name: dattobd -Version: 0.11.5 +Version: 0.11.7 Release: 1%{?dist} Summary: Kernel module and utilities for enabling low-level live backups Vendor: Datto, Inc. @@ -611,13 +611,19 @@ rm %{_systemd_shutdown}/umount_rootfs.shutdown rm %{_systemd_services}/umount-rootfs.service %changelog - * Thu Jan 11 2024 Natalia Zelazna - 0.11.5 - - Fix CentOS7 building +* Fri Feb 09 2024 Natalia Zelazna - 0.11.7 +- Implement tracking bios in dormant state - * Wed Jan 10 2024 Natalia Zelazna - 0.11.4 - - Fix unmounting before shutdown on all distros - - Implement submit_bio-based IO tracking - - Fix CentOS7 building +* Fri Feb 02 2024 Natalia Zelazna - 0.11.6 +- Fix issue with wrong slab allocation flags that led to system reboot + +* Thu Jan 11 2024 Natalia Zelazna - 0.11.5 +- Fix CentOS7 building + +* Wed Jan 10 2024 Natalia Zelazna - 0.11.4 +- Fix unmounting before shutdown on all distros +- Implement submit_bio-based IO tracking +- Fix CentOS7 building * Tue May 19 2023 Lukasz Fulek - 0.11.3 - Fix memory leak on Ubuntu 20.04 diff --git a/src/configure-tests/feature-tests/bio_alloc.c b/src/configure-tests/feature-tests/bio_alloc.c new file mode 100644 index 00000000..49f28a9e --- /dev/null +++ b/src/configure-tests/feature-tests/bio_alloc.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (C) 2024 Kaseya + */ + +// 5.18 <= kernel_version + +#include "includes.h" +MODULE_LICENSE("GPL"); + +static inline void dummy(void){ + struct bio *new_bio; + + new_bio = bio_alloc(GFP_KERNEL, 1); +} diff --git a/src/configure-tests/feature-tests/mmap_write_lock.c b/src/configure-tests/feature-tests/mmap_write_lock.c new file mode 100644 index 00000000..04f59bef --- /dev/null +++ b/src/configure-tests/feature-tests/mmap_write_lock.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* +* Copyright (C) 2024 Kaseya +*/ + +#include "includes.h" +MODULE_LICENSE("GPL"); + +static inline void dummy(void){ + struct mm_struct *mm; + + mmap_write_lock(mm); +} diff --git a/src/configure-tests/symbol-tests b/src/configure-tests/symbol-tests index f0111a24..70ff0995 100644 --- a/src/configure-tests/symbol-tests +++ b/src/configure-tests/symbol-tests @@ -4,3 +4,7 @@ sys_oldumount sys_call_table blk_mq_submit_bio kfree +vm_area_alloc +vm_area_free +insert_vm_struct +vm_area_cachep diff --git a/src/cow_manager.c b/src/cow_manager.c index bdc0dd36..b0f4ad83 100644 --- a/src/cow_manager.c +++ b/src/cow_manager.c @@ -88,17 +88,23 @@ int __cow_alloc_section(struct cow_manager *cm, unsigned long sect_idx, */ int __cow_load_section(struct cow_manager *cm, unsigned long sect_idx) { - int ret; + int ret, i; + int sect_size_bytes = COW_SECTION_SIZE * sizeof(uint64_t); ret = __cow_alloc_section(cm, sect_idx, 0); if (ret) goto error; - ret = file_read(cm->filp, cm->sects[sect_idx].mappings, + for (i = 0; i < sect_size_bytes / COW_BLOCK_SIZE; i++) { + int mapping_offset = (COW_BLOCK_SIZE / sizeof(cm->sects[sect_idx].mappings[0])) * i; + int cow_file_offset = COW_BLOCK_SIZE * i; + + ret = file_read(cm->filp, cm->dev, cm->sects[sect_idx].mappings, cm->sect_size * sect_idx * 8 + COW_HEADER_SIZE, cm->sect_size * 8); if (ret) goto error; + } return 0; @@ -120,15 +126,21 @@ int __cow_load_section(struct cow_manager *cm, unsigned long sect_idx) */ int __cow_write_section(struct cow_manager *cm, unsigned long sect_idx) { - int ret; + int i, ret; + int sect_size_bytes = COW_SECTION_SIZE * sizeof(uint64_t); + + for (i = 0; i < sect_size_bytes / COW_BLOCK_SIZE; i++) { + int mapping_offset = (COW_BLOCK_SIZE / sizeof(cm->sects[sect_idx].mappings[0])) * i; + int cow_file_offset = COW_BLOCK_SIZE * i; - ret = file_write(cm->filp, cm->sects[sect_idx].mappings, + ret = file_write(cm->filp, cm->dev, cm->sects[sect_idx].mappings, cm->sect_size * sect_idx * 8 + COW_HEADER_SIZE, cm->sect_size * 8); if (ret) { LOG_ERROR(ret, "error writing cow manager section to file"); return ret; } + } return 0; } @@ -265,7 +277,7 @@ int __cow_write_header(struct cow_manager *cm, int is_clean) ch.version = cm->version; ch.nr_changed_blocks = cm->nr_changed_blocks; - ret = file_write(cm->filp, &ch, 0, sizeof(struct cow_header)); + ret = file_write(cm->filp, cm->dev, &ch, 0, sizeof(struct cow_header)); if (ret) { LOG_ERROR(ret, "error syncing cow manager header"); return ret; @@ -297,7 +309,7 @@ int __cow_open_header(struct cow_manager *cm, int index_only, int reset_vmalloc) int ret; struct cow_header ch; - ret = file_read(cm->filp, &ch, 0, sizeof(struct cow_header)); + ret = file_read(cm->filp, cm->dev, &ch, 0, sizeof(struct cow_header)); if (ret) goto error; @@ -458,6 +470,9 @@ int cow_sync_and_close(struct cow_manager *cm) if (ret) goto error; + ret = cow_get_file_extents(cm->dev, cm->filp); + if(ret) goto error; + if (cm->filp) file_close(cm->filp); cm->filp = NULL; @@ -569,12 +584,12 @@ int cow_reload(const char *path, uint64_t elements, unsigned long sect_size, cm->allocated_sects = 0; cm->sect_size = sect_size; - cm->log_sect_pages = get_order(sect_size * 8); + cm->log_sect_pages = get_order(sect_size * sizeof(uint64_t)); cm->total_sects = NUM_SEGMENTS(elements, cm->log_sect_pages + PAGE_SHIFT - 3); cm->allowed_sects = __cow_calculate_allowed_sects(cache_size, cm->total_sects); - cm->data_offset = COW_HEADER_SIZE + (cm->total_sects * (sect_size * 8)); + cm->data_offset = COW_HEADER_SIZE + (cm->total_sects * (sect_size * sizeof(uint64_t))); ret = __cow_open_header(cm, index_only, 1); if (ret) @@ -701,7 +716,7 @@ int cow_init(const char *path, uint64_t elements, unsigned long sect_size, LOG_DEBUG("allocating cow file (%llu bytes)", (unsigned long long)file_max); - ret = file_allocate(cm->filp, 0, file_max); + ret = file_allocate(cm->filp, cm->dev, 0, file_max); if (ret) goto error; @@ -811,6 +826,35 @@ int cow_read_mapping(struct cow_manager *cm, uint64_t pos, uint64_t *out) return ret; } + +/** + * cow_block_has_data() - + * + * @cm: The &struct cow_manager associated with the &struct snap_device. + * @pos: The section index offset within the cache. + * + * Return: + * * false - pos does not have data in "memory" + * * true - pos has data in "memory" + */ +bool cow_block_has_data(struct cow_manager *cm, uint64_t pos) +{ + uint64_t sect_idx = pos; + uint64_t block_mapping; + unsigned long sect_pos = do_div(sect_idx, cm->sect_size); + + if (!cm->sects[sect_idx].mappings) { + return false; + } + block_mapping = cm->sects[sect_idx].mappings[sect_pos]; + if(block_mapping) return true; + return false; + +} + + + + /** * __cow_write_mapping() - Writes the specified section to the COW file. * @@ -896,7 +940,7 @@ static int __cow_write_data(struct cow_manager *cm, void *buf) goto error; } - ret = file_write(cm->filp, buf, curr_size, COW_BLOCK_SIZE); + ret = file_write(cm->filp, cm->dev, buf, curr_size, COW_BLOCK_SIZE); if (ret) goto error; @@ -975,7 +1019,7 @@ int cow_read_data(struct cow_manager *cm, void *buf, uint64_t block_pos, if (block_off >= COW_BLOCK_SIZE) return -EINVAL; - ret = file_read(cm->filp, buf, (block_pos * COW_BLOCK_SIZE) + block_off, + ret = file_read(cm->filp, cm->dev, buf, (block_pos * COW_BLOCK_SIZE) + block_off, len); if (ret) { LOG_ERROR(ret, "error reading cow data"); @@ -984,3 +1028,137 @@ int cow_read_data(struct cow_manager *cm, void *buf, uint64_t block_pos, return 0; } + +int cow_get_file_extents(struct snap_device* dev, struct file* filp) +{ + int ret; + struct fiemap_extent_info fiemap_info; + unsigned int fiemap_mapped_extents_size, i_ext; + struct fiemap_extent *extent; + char parent_process_name[TASK_COMM_LEN]; + unsigned long vm_flags = VM_READ | VM_WRITE; + unsigned long start_addr; + struct task_struct *task; + struct vm_area_struct *vma; + struct page *pg; + __user uint8_t *cow_ext_buf; + + unsigned long cow_ext_buf_size = ALIGN(dattobd_cow_ext_buf_size, PAGE_SIZE); + + int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); + + int (*insert_vm_struct)(struct mm_struct *mm, struct vm_area_struct *vma) = (INSERT_VM_STRUCT_ADDR != 0) ? + (int (*)(struct mm_struct *mm, struct vm_area_struct *vma)) (INSERT_VM_STRUCT_ADDR + (long long)(((void *)kfree) - (void *)KFREE_ADDR)) : NULL; + + if (!insert_vm_struct) { + LOG_ERROR(-ENOTSUPP, "insert_vm_struct() was not found"); + return -ENOTSUPP; + } + + fiemap = NULL; + task = get_current(); + + LOG_DEBUG("getting cow file extents from filp=%p", filp); + LOG_DEBUG("attempting page stealing from %s", get_task_comm(parent_process_name, task)); + + dattobd_mm_lock(task->mm); + start_addr = get_unmapped_area(NULL, 0, cow_ext_buf_size, 0, VM_READ | VM_WRITE); + + if (IS_ERR_VALUE(start_addr)) + return start_addr; // returns -EPERM if failed + + + vma = dattobd_vm_area_allocate(task->mm); + + if (!vma) { + ret = -ENOMEM; + LOG_ERROR(ret, "vm_area_alloc() failed"); + dattobd_mm_unlock(task->mm); + return ret; + } + + vma->vm_start = start_addr; + vma->vm_end = start_addr + cow_ext_buf_size; + *(unsigned long *) &vma->vm_flags = vm_flags; + vma->vm_page_prot = vm_get_page_prot(vm_flags); + vma->vm_pgoff = 0; + + ret = insert_vm_struct(task->mm, vma); + if (ret < 0) { + ret = -EINVAL; + LOG_ERROR(ret, "insert_vm_struct() failed"); + dattobd_vm_area_free(vma); + dattobd_mm_unlock(task->mm); + return ret; + } + + pg = alloc_pages(GFP_USER, get_order(cow_ext_buf_size)); + if (!pg) { + ret = -ENOMEM; + LOG_ERROR(ret, "alloc_page() failed"); + dattobd_vm_area_free(vma); + dattobd_mm_unlock(task->mm); + return ret; + } + + SetPageReserved(pg); + ret = remap_pfn_range(vma, vma->vm_start, page_to_pfn(pg), cow_ext_buf_size, PAGE_SHARED); + if (ret < 0) { + LOG_ERROR(ret, "remap_pfn_range() failed"); + ClearPageReserved(pg); + __free_pages(pg, get_order(cow_ext_buf_size)); + dattobd_vm_area_free(vma); + dattobd_mm_unlock(task->mm); + return ret; + } + + cow_ext_buf = (__user uint8_t *) start_addr; + + if (filp->f_inode->i_op) + fiemap = filp->f_inode->i_op->fiemap; + + if (fiemap) { + int64_t fiemap_max = ~0ULL & ~(1ULL << 63); + int max_num_extents = cow_ext_buf_size; // used for do_div() as it overwrites the first argument + + fiemap_info.fi_flags = FIEMAP_FLAG_SYNC; + fiemap_info.fi_extents_mapped = 0; + do_div(max_num_extents, sizeof(struct fiemap_extent)); + fiemap_info.fi_extents_max = max_num_extents; + fiemap_info.fi_extents_start = (struct fiemap_extent __user *)cow_ext_buf; + + ret = fiemap(filp->f_inode, &fiemap_info, 0, fiemap_max); + + LOG_DEBUG("fiemap for cow file (ret %d), extents %u (max %u)", ret, + fiemap_info.fi_extents_mapped, fiemap_info.fi_extents_max); + + if (!ret && fiemap_info.fi_extents_mapped > 0) { + if (dev->sd_cow_extents) kfree(dev->sd_cow_extents); + fiemap_mapped_extents_size = fiemap_info.fi_extents_mapped * sizeof(struct fiemap_extent); + dev->sd_cow_extents = kmalloc(fiemap_mapped_extents_size, GFP_KERNEL); + if (dev->sd_cow_extents) { + //TODO: closely watch + ret = copy_from_user(dev->sd_cow_extents, cow_ext_buf, fiemap_mapped_extents_size); + if (!ret) { + dev->sd_cow_ext_cnt = fiemap_info.fi_extents_mapped; + WARN(dev->sd_cow_ext_cnt == max_num_extents, "max num of extents read, increase cow_ext_buf_size"); + extent = dev->sd_cow_extents; + for (i_ext = 0; i_ext < fiemap_info.fi_extents_mapped; ++i_ext, ++extent) { + LOG_DEBUG(" cow file extent: log 0x%llx, phy 0x%llx, len %llu", extent->fe_logical, extent->fe_physical, extent->fe_length); + } + } + } + } + } else { + ret = -ENOTSUPP; + LOG_ERROR(ret, "fiemap not supported"); + goto out; + } + +out: + ClearPageReserved(pg); + dattobd_mm_unlock(task->mm); + vm_munmap(vma->vm_start, cow_ext_buf_size); + __free_pages(pg, get_order(cow_ext_buf_size)); + return ret; +} diff --git a/src/cow_manager.h b/src/cow_manager.h index 5dca5c34..33a83849 100644 --- a/src/cow_manager.h +++ b/src/cow_manager.h @@ -17,6 +17,7 @@ #define COW_SECTION_SIZE 4096 #define cow_write_filler_mapping(cm, pos) __cow_write_mapping(cm, pos, 1) +static unsigned long dattobd_cow_ext_buf_size = sizeof(struct fiemap_extent) * 1024; /** * struct cow_section - maintains data and usage statistics for a cow section. @@ -61,6 +62,7 @@ struct cow_manager { // be allocated at once struct cow_section *sects; // pointer to the array of sections of // mappings + struct snap_device* dev; //pointer to snapshot device }; /***************************COW MANAGER FUNCTIONS**************************/ @@ -94,6 +96,10 @@ int cow_write_current(struct cow_manager *cm, uint64_t block, void *buf); int cow_read_data(struct cow_manager *cm, void *buf, uint64_t block_pos, unsigned long block_off, unsigned long len); +bool cow_block_has_data(struct cow_manager *cm, uint64_t pos); + int __cow_write_mapping(struct cow_manager *cm, uint64_t pos, uint64_t val); +int cow_get_file_extents(struct snap_device* dev, struct file* filp); + #endif /* COW_MANAGER_H_ */ diff --git a/src/dattobd.h b/src/dattobd.h index 90662843..cabf0e80 100644 --- a/src/dattobd.h +++ b/src/dattobd.h @@ -15,7 +15,7 @@ #include #include -#define DATTOBD_VERSION "0.11.5" +#define DATTOBD_VERSION "0.11.7" #define DATTO_IOCTL_MAGIC 0x91 struct setup_params { diff --git a/src/filesystem.c b/src/filesystem.c index 7d68249c..ef157a85 100644 --- a/src/filesystem.c +++ b/src/filesystem.c @@ -8,6 +8,7 @@ #include "includes.h" #include "logging.h" #include "userspace_copy_helpers.h" +#include "snap_device.h" // if this isn't defined, we don't need it anyway #ifndef FMODE_NONOTIFY @@ -44,23 +45,37 @@ static int kern_path(const char *name, unsigned int flags, struct path *path) * * Return: The number of bytes read or a negative errno. */ -static ssize_t dattobd_kernel_read(struct file *filp, void *buf, size_t count, +static ssize_t dattobd_kernel_read(struct file *filp, struct snap_device* dev, void *buf, size_t count, loff_t *pos) { + ssize_t ret; + + if(filp){ #ifndef HAVE_KERNEL_READ_PPOS //#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) mm_segment_t old_fs; - ssize_t ret; + file_unlock(filp); old_fs = get_fs(); set_fs(get_ds()); ret = vfs_read(filp, (char __user *)buf, count, pos); set_fs(old_fs); - + file_lock(filp); return ret; #else - return kernel_read(filp, buf, count, pos); + file_unlock(filp); + ret=kernel_read(filp, buf, count, pos); + file_lock(filp); + return ret; #endif + }else{ + LOG_DEBUG("DIO: reading %lu sectors...", count / SECTOR_SIZE); + + ret = file_read_block(dev, buf, *pos, count / SECTOR_SIZE); + if (!ret) ret = count; + + return ret; + } } /** @@ -74,13 +89,15 @@ static ssize_t dattobd_kernel_read(struct file *filp, void *buf, size_t count, * * Return: The number of bytes written or a negative errno. */ -static ssize_t dattobd_kernel_write(struct file *filp, const void *buf, +static ssize_t dattobd_kernel_write(struct file *filp,struct snap_device* dev, const void *buf, size_t count, loff_t *pos) { + ssize_t ret; + + if(filp){ #ifndef HAVE_KERNEL_WRITE_PPOS //#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0) mm_segment_t old_fs; - ssize_t ret; old_fs = get_fs(); set_fs(get_ds()); @@ -89,14 +106,25 @@ static ssize_t dattobd_kernel_write(struct file *filp, const void *buf, return ret; #else - return kernel_write(filp, buf, count, pos); + file_unlock(filp); + ret= kernel_write(filp, buf, count, pos); + file_lock(filp); + return ret; #endif + }else{ + LOG_DEBUG("DIO: writing %lu sectors...", count / SECTOR_SIZE); + + ret = file_write_block(dev, buf, *pos, count / SECTOR_SIZE); + if (!ret) ret = count; + + return ret; + } } /** * file_io() - Reads or writes to the supplied file. * - * @filp: A pointer to the file object. + * @cm: A pointer to the cow manager * @is_write: An integer encoded bool indicating a write or read operation. * @buf: Input/output buffer for write/read, respectively. * @offset: Byte offset of the first sequential access within @filp. @@ -106,16 +134,16 @@ static ssize_t dattobd_kernel_write(struct file *filp, const void *buf, * * 0 - success * * !0 - errno indicating the error */ -int file_io(struct file *filp, int is_write, void *buf, sector_t offset, +int file_io(struct file *filp, struct snap_device* dev, int is_write, void *buf, sector_t offset, unsigned long len) { ssize_t ret; loff_t off = (loff_t)offset; if (is_write) - ret = dattobd_kernel_write(filp, buf, len, &off); + ret = dattobd_kernel_write(filp, dev, buf, len, &off); else - ret = dattobd_kernel_read(filp, buf, len, &off); + ret = dattobd_kernel_read(filp, dev, buf, len, &off); if (ret < 0) { LOG_ERROR((int)ret, "error performing file '%s': %llu, %lu", @@ -137,7 +165,7 @@ int file_io(struct file *filp, int is_write, void *buf, sector_t offset, * file_write() - Writes @len bytes of data to offset @offset within @filp from * @buf. * - * @filp: A pointer to the file object. + * @cm: A pointer to the cow manager object. * @buf: Input buffer for write. * @offset: Byte offset of the first sequential access within @filp. * @len: The number of bytes in the transfer. @@ -146,13 +174,13 @@ int file_io(struct file *filp, int is_write, void *buf, sector_t offset, * * 0 - success * * !0 - errno indicating the error */ -#define file_write(filp, buf, offset, len) file_io(filp, 1, buf, offset, len) +//#define file_write(filp, dev, buf, offset, len) file_io(filp, dev, 1, buf, offset, len) /** * file_read() - Store @len bytes of data from offset @offset within @filp into * @buf. * - * @filp: A pointer to the file object. + * @cm: A pointer to the cow manager object. * @buf: Output buffer for read. * @offset: Byte offset of the first sequential access within @filp. * @len: The number of bytes in the transfer. @@ -161,7 +189,7 @@ int file_io(struct file *filp, int is_write, void *buf, sector_t offset, * * 0 - success * * !0 - errno indicating the error */ -#define file_read(filp, buf, offset, len) file_io(filp, 0, buf, offset, len) +//#define file_read(filp, dev, buf, offset, len) file_io(filp, dev, 0, buf, offset, len) /** * file_close() - Closes the file object. @@ -170,6 +198,7 @@ int file_io(struct file *filp, int is_write, void *buf, sector_t offset, */ inline void file_close(struct file *f) { + file_unlock_mark_dirty(f); filp_close(f, NULL); } @@ -765,7 +794,7 @@ static int real_fallocate(struct file *f, uint64_t offset, uint64_t length) * within the range specified by @offset and @length. Attempts to use * @real_fallocate with a fallback of writing zeroes if that fails. * - * @f: A &struct file object. + * @cm: A &struct cow_manager object. * @offset: The offset into @f indicating the start of the allocation. * @length: The number of byte to allocate starting at @offset. * @@ -773,7 +802,7 @@ static int real_fallocate(struct file *f, uint64_t offset, uint64_t length) * * 0 - success * * !0 - errno indicating the error. */ -int file_allocate(struct file *f, uint64_t offset, uint64_t length) +int file_allocate(struct file *filp, struct snap_device* dev, uint64_t offset, uint64_t length) { int ret = 0; char *page_buf = NULL; @@ -781,25 +810,7 @@ int file_allocate(struct file *f, uint64_t offset, uint64_t length) char *abs_path = NULL; int abs_path_len; - file_get_absolute_pathname(f, &abs_path, &abs_path_len); - - // try regular fallocate - ret = real_fallocate(f, offset, length); - if (ret && ret != -EOPNOTSUPP) - goto error; - else if (!ret) - goto out; - - // fallocate isn't supported, fall back on writing zeros - if (!abs_path) { - LOG_WARN("fallocate is not supported for this file system, " - "falling back on " - "writing zeros"); - } else { - LOG_WARN("fallocate is not supported for '%s', falling back on " - "writing zeros", - abs_path); - } + file_get_absolute_pathname(filp, &abs_path, &abs_path_len); // allocate page of zeros page_buf = (char *)get_zeroed_page(GFP_KERNEL); @@ -814,7 +825,7 @@ int file_allocate(struct file *f, uint64_t offset, uint64_t length) // if not page aligned, write zeros to that point if (offset % PAGE_SIZE != 0) { - ret = file_write(f, page_buf, offset, + ret = file_write(filp, dev, page_buf, offset, PAGE_SIZE - (offset % PAGE_SIZE)); if (ret) goto error; @@ -824,11 +835,12 @@ int file_allocate(struct file *f, uint64_t offset, uint64_t length) // write a page of zeros at a time for (i = 0; i < write_count; i++) { - ret = file_write(f, page_buf, offset + (PAGE_SIZE * i), + ret = file_write(filp, dev, page_buf, offset + (PAGE_SIZE * i), PAGE_SIZE); if (ret) goto error; } + file_lock(filp); out: if (page_buf) @@ -988,5 +1000,306 @@ void dattobd_inode_unlock(struct inode *inode) { mutex_unlock(&inode->i_mutex); } +#endif + +struct kmem_cache **vm_area_cache = (VM_AREA_CACHEP_ADDR != 0) ? + (struct kmem_cache **) (VM_AREA_CACHEP_ADDR + (long long)(((void *)kfree) - (void *)KFREE_ADDR)) : NULL; + +struct vm_area_struct* dattobd_vm_area_allocate(struct mm_struct* mm) +{ + struct vm_area_struct *vma; + static const struct vm_operations_struct dummy_vm_ops = {}; + + if (!vm_area_cache) { + LOG_ERROR(-ENOTSUPP, "vm_area_cachep was not found"); + return NULL; + } + vma = kmem_cache_zalloc(*vm_area_cache, GFP_KERNEL); + if (!vma) { + LOG_ERROR(-ENOMEM, "kmem_cache_zalloc() failed"); + return NULL; + } + + vma->vm_mm = mm; + vma->vm_ops = &dummy_vm_ops; + INIT_LIST_HEAD(&vma->anon_vma_chain); + return vma; +} + +void dattobd_vm_area_free(struct vm_area_struct *vma) +{ + kmem_cache_free(*vm_area_cache, vma); +} + +void dattobd_mm_lock(struct mm_struct* mm) +{ +#ifdef HAVE_MMAP_WRITE_LOCK + mmap_write_lock(mm); +#else + down_write(&mm->mmap_sem); +#endif +} + +void dattobd_mm_unlock(struct mm_struct* mm) +{ +#ifdef HAVE_MMAP_WRITE_LOCK + mmap_write_unlock(mm); +#else + up_write(&mm->mmap_sem); +#endif +} + +void file_switch_lock(struct file* filp, bool lock, bool mark_dirty) +{ + struct inode* inode; + + if(!filp) return; + + inode= dattobd_get_dentry(filp)->d_inode; + igrab(inode); + + if(lock){ + inode->i_flags |= S_IMMUTABLE; + }else{ + inode->i_flags &= ~S_IMMUTABLE; + } + + if(mark_dirty){ + mark_inode_dirty(inode); + } + + iput(inode); +} +int file_write_block(struct snap_device* dev, const void* block, size_t offset, size_t len) +{ + int ret; + int bytes; + char *data; + struct page *pg; + struct bio_set *bs; + struct bio *new_bio; + struct block_device *bdev; + sector_t start_sect; + int sectors_processed; + int iterations_done; + int bytes_written; + + ret = 0; + bs = dev_bioset(dev); + bdev = dev->sd_base_dev; + sectors_processed = 0; + +write_bio: + start_sect = sector_by_offset(dev, offset); + if (start_sect == SECTOR_INVALID) { + LOG_WARN("Possible write IO to the end of file (offset=%lu)", offset); + ret = -EFAULT; + goto out; + } + +#ifdef HAVE_BIO_ALLOC + new_bio = bio_alloc(GFP_NOIO, 1); +#else + new_bio = bio_alloc(bdev, 1, 0, GFP_KERNEL); #endif + if(!new_bio){ + ret = -ENOMEM; + LOG_ERROR(ret, "error allocating bio (write) - bs = %p", bs); + goto out; + } + + dattobd_bio_set_dev(new_bio, bdev); + dattobd_set_bio_ops(new_bio, REQ_OP_READ, 0); + //from bio_helper.h + bio_sector(new_bio) = start_sect; + bio_idx(new_bio) = 0; + + pg = alloc_page(GFP_NOIO); + if(!pg){ + ret = -ENOMEM; + LOG_ERROR(ret, "error allocating read bio page"); + goto out; + } + + data = kmap(pg); + iterations_done = 0; + bytes_written = 0; + + do { + bytes_written = iterations_done * SECTOR_SIZE; + memcpy(data + bytes_written, block + sectors_processed * SECTOR_SIZE, SECTOR_SIZE); + offset += SECTOR_SIZE; + sectors_processed++; + iterations_done++; + } while (sectors_processed < len && + sector_by_offset(dev, offset) == start_sect + iterations_done); + + kunmap(pg); + + bytes_written = iterations_done * SECTOR_SIZE; + bytes = bio_add_page(new_bio, pg, bytes_written, 0); + if(bytes != bytes_written){ + LOG_DEBUG("bio_add_page() error!"); + __free_page(pg); + ret = -EFAULT; + goto out; + } + + if (dev->sd_cow_inode) + pg->mapping = dev->sd_cow_inode->i_mapping; + + + ret = dattobd_submit_bio_wait(new_bio); + if (ret) { + LOG_ERROR(ret, "submit_bio_wait() error!"); + goto out; + } + + pg->mapping = NULL; + bio_free_clone(new_bio); + new_bio = NULL; + + if (sectors_processed != len) + goto write_bio; + +out: + if (new_bio) { + pg->mapping = NULL; + bio_free_clone(new_bio); + } + + return ret; +} + +int file_read_block(struct snap_device* dev, void* block, size_t offset, size_t len) +{ + int ret; + int bytes; + struct page *pg; + struct bio_set *bs; + struct bio *new_bio; + struct block_device *bdev; + sector_t start_sect; + struct bio_vec *bvec; +#ifdef HAVE_BVEC_ITER_ALL + struct bvec_iter_all iter; +#else + int i = 0; +#endif + int sectors_processed; + int iterations_done; + int bytes_to_read; + int buf_offset; + + ret = 0; + bs = dev_bioset(dev); + bdev = dev->sd_base_dev; + sectors_processed = 0; +WARN_ON(len > SECTORS_PER_BLOCK); + +read_bio: + start_sect = sector_by_offset(dev, offset); + if (start_sect == SECTOR_INVALID) { + LOG_WARN("Possible read IO to the end of file (offset=%lu)", offset); + ret = -EFAULT; + goto out; + } +#ifdef HAVE_BIO_ALLOC + new_bio = bio_alloc(GFP_NOIO, 1); +#else + new_bio = bio_alloc(bdev, 1, 0, GFP_KERNEL); +#endif + if(!new_bio){ + ret = -ENOMEM; + LOG_ERROR(ret, "error allocating bio (read) - bs = %p", bs); + goto out; + } + dattobd_bio_set_dev(new_bio, bdev); + dattobd_set_bio_ops(new_bio, REQ_OP_READ, 0); + bio_sector(new_bio) = start_sect; + bio_idx(new_bio) = 0; + + //allocate a page and add it to our bio + pg = alloc_page(GFP_NOIO); + if(!pg){ + ret = -ENOMEM; + LOG_ERROR(ret, "error allocating read bio page"); + goto out; + } + + iterations_done = 0; + bytes_to_read = 0; + buf_offset = sectors_processed * SECTOR_SIZE; + + do { + offset += SECTOR_SIZE; + sectors_processed++; + iterations_done++; + } while (sectors_processed < len && + sector_by_offset(dev, offset) == start_sect + iterations_done); + + bytes_to_read = iterations_done * SECTOR_SIZE; + bytes = bio_add_page(new_bio, pg, bytes_to_read, 0); + if(bytes != bytes_to_read){ + LOG_DEBUG("bio_add_page() error!"); + __free_page(pg); + ret = -EFAULT; + goto out; + } + + if (dev->sd_cow_inode) + pg->mapping = dev->sd_cow_inode->i_mapping; + + ret = dattobd_submit_bio_wait(new_bio); + if (ret) { + LOG_ERROR(ret, "submit_bio_wait() error!"); + goto out; + } + +#ifdef HAVE_BVEC_ITER_ALL + bio_for_each_segment_all(bvec, new_bio, iter) { +#else + bio_for_each_segment_all(bvec, new_bio, i) { +#endif + struct page *pg = bvec->bv_page; + char *data = kmap(pg); + WARN_ON(bytes_to_read != bvec->bv_len); + memcpy(block + buf_offset, data, bytes_to_read); + kunmap(pg); + // in an impossible case if we have more + // than one page (should never happen) + break; + } + + + + pg->mapping = NULL; + bio_free_clone(new_bio); + new_bio = NULL; + + if (sectors_processed != len) + goto read_bio; + +out: + if (new_bio) { + pg->mapping = NULL; + bio_free_clone(new_bio); + } + + return ret; +} + +sector_t sector_by_offset(struct snap_device*dev, size_t offset) +{ + unsigned int i; + struct fiemap_extent *extent = dev->sd_cow_extents; + for (i = 0; i < dev->sd_cow_ext_cnt; i++) { + if (offset >= extent[i].fe_logical && offset < extent[i].fe_logical + extent[i].fe_length) + return (extent[i].fe_physical + (offset - extent[i].fe_logical)) >> 9; + } + + return SECTOR_INVALID; +} + + diff --git a/src/filesystem.h b/src/filesystem.h index d1304129..02df9788 100644 --- a/src/filesystem.h +++ b/src/filesystem.h @@ -9,14 +9,19 @@ #include "includes.h" #include "userspace_copy_helpers.h" +#include "snap_device.h" -#define file_write(filp, buf, offset, len) file_io(filp, 1, buf, offset, len) -#define file_read(filp, buf, offset, len) file_io(filp, 0, buf, offset, len) +#define file_write(filp, dev, buf, offset, len) file_io(filp, dev, 1, buf, offset, len) +#define file_read(filp, dev, buf, offset, len) file_io(filp, dev, 0, buf, offset, len) #define file_unlink(filp) __file_unlink(filp, 0, 0) #define file_unlink_and_close(filp) __file_unlink(filp, 1, 0) #define file_unlink_and_close_force(filp) __file_unlink(filp, 1, 1) +#define file_lock(filp) file_switch_lock(filp, true, false) +#define file_unlock(filp) file_switch_lock(filp, false, false) +#define file_unlock_mark_dirty(filp) file_switch_lock(filp, false, true) + #ifndef HAVE_STRUCT_PATH //#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20) #define dattobd_get_dentry(f) (f)->f_dentry @@ -40,6 +45,7 @@ // takes a value and the log of the value it should be rounded up to #define NUM_SEGMENTS(x, log_size) (((x) + (1 << (log_size)) - 1) >> (log_size)) +#define SECTOR_INVALID ~(u64)0 struct file; struct dentry; @@ -57,7 +63,7 @@ struct path { typedef mode_t fmode_t; #endif -int file_io(struct file *filp, int is_write, void *buf, sector_t offset, +int file_io(struct file *filp, struct snap_device* dev, int is_write, void *buf, sector_t offset, unsigned long len); void file_close(struct file *f); @@ -85,7 +91,7 @@ int user_mount_pathname_concat(const char __user *user_mount_path, int file_truncate(struct file *filp, loff_t len); -int file_allocate(struct file *f, uint64_t offset, uint64_t length); +int file_allocate(struct file *filp, struct snap_device* dev, uint64_t offset, uint64_t length); int __file_unlink(struct file *filp, int close, int force); @@ -109,4 +115,20 @@ void dattobd_inode_unlock(struct inode *inode); #define dattobd_inode_unlock inode_unlock #endif +struct vm_area_struct* dattobd_vm_area_allocate(struct mm_struct* mm); + +void dattobd_vm_area_free(struct vm_area_struct *vma); + +void dattobd_mm_lock(struct mm_struct* mm); + +void dattobd_mm_unlock(struct mm_struct* mm); + +void file_switch_lock(struct file* filp, bool lock, bool mark_dirty); + +int file_write_block(struct snap_device* dev, const void* block, size_t offset, size_t len); + +int file_read_block(struct snap_device* dev, void* block, size_t offset, size_t len); + +sector_t sector_by_offset(struct snap_device*dev, size_t offset); + #endif /* FILESYSTEM_H_ */ diff --git a/src/includes.h b/src/includes.h index f5d05bd8..11b6c226 100644 --- a/src/includes.h +++ b/src/includes.h @@ -23,5 +23,6 @@ #include #include #include +#include #endif diff --git a/src/module_threads.c b/src/module_threads.c index 149eff67..334a64a4 100644 --- a/src/module_threads.c +++ b/src/module_threads.c @@ -119,12 +119,6 @@ int snap_cow_thread(void *data) cow_free_members(dev->sd_cow); } - int should_stop=kthread_should_stop(); - if(should_stop){ - LOG_DEBUG("stopping snap thread in if"); - break; - } - if (bio_queue_empty(bq)) continue; diff --git a/src/snap_device.h b/src/snap_device.h index 9ed33aa1..28d6590c 100644 --- a/src/snap_device.h +++ b/src/snap_device.h @@ -64,6 +64,8 @@ struct snap_device { // read/writes struct bio_queue sd_orig_bios; // list of outstanding original bios struct sset_queue sd_pending_ssets; // list of outstanding sector sets + struct fiemap_extent *sd_cow_extents; //cow file extents + unsigned int sd_cow_ext_cnt; //cow file extents count #ifndef HAVE_BIOSET_INIT //#if LINUX_VERSION_CODE < KERNEL_VERSION(4,18,0) struct bio_set *sd_bioset; // allocation pool for bios diff --git a/src/snap_handle.c b/src/snap_handle.c index 530a9c5b..6bd8fe7b 100644 --- a/src/snap_handle.c +++ b/src/snap_handle.c @@ -261,7 +261,7 @@ int snap_handle_write_bio(const struct snap_device *dev, struct bio *bio) // find the start and end block start_block = end_block; - end_block = start_block + bvec->bv_len / COW_BLOCK_SIZE; + end_block = start_block + DIV_ROUND_UP(bvec->bv_len, COW_BLOCK_SIZE); // map the page into kernel space data = kmap(bvec->bv_page); diff --git a/src/tracer.c b/src/tracer.c index dfe648f7..5da64dd3 100644 --- a/src/tracer.c +++ b/src/tracer.c @@ -163,6 +163,32 @@ void dattobd_free_request_tracking_ptr(struct snap_device *dev) #endif } +/** + * bio_is_first_write() - + * + * @dev: The &struct snap_device that keeps device state. + * @bio: The &struct bio which describes the I/O. + * + * Return: + * * 0 - is not first write + * * 1 - is first write + */ +static bool bio_is_first_write(struct snap_device *dev, struct bio *bio) +{ + sector_t sector_start = bio_sector(bio) - dev->sd_sect_off; + sector_t sector_end = sector_start + (bio_size(bio) / SECTOR_SIZE); + sector_t block_start = SECTOR_TO_BLOCK(sector_start); + sector_t block_end = SECTOR_TO_BLOCK(sector_end); + + for(; block_start <= block_end; block_start++){ + //if any block in the bio has no data, then it is a first write + if (!cow_block_has_data(dev->sd_cow, block_start)) return true; + } + return false; + +} + + /** * snap_trace_bio() - Traces a bio when snapshotting. For bio reads there is * nothing to do and the request is passed to the original driver. For writes @@ -196,6 +222,20 @@ static int snap_trace_bio(struct snap_device *dev, struct bio *bio) #endif } + if(dev->sd_cow && dev->sd_cow->sects){ + // if bio is not first write just call original mrf, we dont need to do make read clone + if (!bio_is_first_write(dev, bio)){ +#ifdef HAVE_NONVOID_SUBMIT_BIO_1 + return SUBMIT_BIO_REAL(dev, bio); +#else + SUBMIT_BIO_REAL(dev, bio); + return 0; +#endif + } + + } + + // the cow manager works in 4096 byte blocks, so read clones must also // be 4096 byte aligned start_sect = ROUND_DOWN(bio_sector(bio) - dev->sd_sect_off, @@ -561,6 +601,19 @@ static int __tracer_destroy_cow(struct snap_device *dev, int close_method) } } + if (close_method != 2 && dev->sd_cow_extents) { + LOG_DEBUG("destroying cow file extents"); + kfree(dev->sd_cow_extents); + dev->sd_cow_extents = NULL; + dev->sd_cow_ext_cnt = 0; + dev->sd_cow_inode = NULL; + } else { + LOG_DEBUG("preserving cow file extents"); + } + + dev->sd_falloc_size = 0; + dev->sd_cache_size = 0; + return ret; } @@ -844,6 +897,10 @@ static void __tracer_copy_cow(const struct snap_device *src, struct snap_device *dest) { dest->sd_cow = src->sd_cow; + // copy cow file extents and update the device + dest->sd_cow_extents = src->sd_cow_extents; + dest->sd_cow_ext_cnt = src->sd_cow_ext_cnt; + dest->sd_cow_inode = src->sd_cow_inode; dest->sd_cache_size = src->sd_cache_size; dest->sd_falloc_size = src->sd_falloc_size; @@ -1577,6 +1634,26 @@ static void __tracer_destroy_tracing(struct snap_device *dev) LOG_DEBUG("replacing make_request_fn if needed"); if(__tracer_should_reset_mrf(dev)){ LOG_DEBUG("__tracer_should_reset_mrf is true"); + + if (!test_bit(ACTIVE, &dev->sd_state)) { + int ret = 0; + LOG_DEBUG("flushing bio requests"); + + if (!test_bit(SNAPSHOT, &dev->sd_state)) { + ret = __tracer_setup_inc_cow_thread(dev, dev->sd_minor); + } else { + ret = __tracer_setup_snap_cow_thread(dev, dev->sd_minor); + } + + if(ret) { + LOG_ERROR(ret, "Failed to setup cow thread for device with minor %i and flush bio requests", dev->sd_minor); + } + + wake_up_process(dev->sd_cow_thread); + //TODO: Maybe some waiting mechanism will be needed + __tracer_destroy_cow_thread(dev); + } + #ifndef USE_BDOPS_SUBMIT_BIO __tracer_transition_tracing( NULL,