diff --git a/zfs-win/BlockReader.cpp b/zfs-win/BlockReader.cpp index c928c18..0dd462a 100644 --- a/zfs-win/BlockReader.cpp +++ b/zfs-win/BlockReader.cpp @@ -21,365 +21,179 @@ #include "stdafx.h" #include "BlockReader.h" -#include "Hash.h" -#include "Compress.h" namespace ZFS { - BlockReader::BlockReader(Pool* pool, blkptr_t* bp, size_t count) + BlockReader::BlockReader(Pool* pool, dnode_phys_t* dn) : m_pool(pool) { - Insert(bp, count); + m_node = *dn; + m_datablksize = m_node.datablkszsec << 9; + m_indblksize = 1 << m_node.indblkshift; + m_indblkcount = m_indblksize / sizeof(blkptr_t); + m_size = (m_node.maxblkid + 1) * m_datablksize; + m_cache.id = -1; + + ASSERT(m_node.nlevels > 0); + ASSERT(m_node.indblkshift >= 7); + ASSERT(m_node.nblkptr <= m_indblkcount); + + m_tree.resize(m_node.nlevels); + + size_t n = (size_t)((m_node.maxblkid + 1 + m_indblkcount - 1) / m_indblkcount); + + for(size_t i = 0; i < m_tree.size(); i++) + { + blklvl_t& lvl = m_tree[i]; + + lvl.resize(n); + + memset(lvl.data(), 0, lvl.size() * sizeof(blkcol_t*)); + + n = (n + m_indblkcount - 1) / m_indblkcount; + } + + blkcol_t* col = new blkcol_t(m_node.nblkptr); + + memcpy(col->data(), m_node.blkptr, m_node.nblkptr * sizeof(blkptr_t)); + + m_tree.back()[0] = col; } BlockReader::~BlockReader() { - for(auto i = m_bpl.begin(); i != m_bpl.end(); i++) + for(auto i = m_tree.begin(); i != m_tree.end(); i++) { - delete *i; - } - } - - void BlockReader::Insert(blkptr_t* bp, size_t count) - { - if(m_bpl.empty()) - { - for(size_t i = 0; i < count; i++) + for(auto j = i->begin(); j != i->end(); j++) { - if(bp[i].type != DMU_OT_NONE) - { - m_bpl.push_back(new blkptr_t(bp[i])); - } - else - { - ASSERT(bp[i].lsize == 0); - } + delete *j; } } - else - { - std::list l; - - for(size_t i = 0; i < count; i++) - { - if(bp[i].type != DMU_OT_NONE) - { - l.push_back(new blkptr_t(bp[i])); - } - else - { - ASSERT(bp[i].lsize == 0); - } - } - - m_bpl.insert(m_bpl.begin(), l.begin(), l.end()); - } } - bool BlockReader::Read(std::vector& dst, blkptr_t* bp) + size_t BlockReader::Read(void* dst, size_t size, uint64_t offset) { - for(int i = 0; i < 3; i++) + uint64_t block_id = offset / m_datablksize; + size_t block_offset = (size_t)(offset - (uint64_t)m_datablksize * block_id); + + uint8_t* ptr = (uint8_t*)dst; + + for(; block_id <= m_node.maxblkid && size > 0; block_id++) { - dva_t* addr = &bp->blk_dva[i]; + blkptr_t* bp = NULL; - ASSERT(addr->gang == 0); // TODO: zio_gbh_phys_t (not used ??? never encountered one, yet) - - for(auto i = m_pool->m_vdevs.begin(); i != m_pool->m_vdevs.end(); i++) - { - VirtualDevice* vdev = *i; - - if(vdev->id != addr->vdev) - { - continue; - } - - // << 9 or vdev->ashift? - // - // on-disk spec says: "All sizes are stored as the number of 512 byte sectors (minus one) needed to - // represent the size of this block.", but is it still true or outdated? - - size_t psize = ((size_t)bp->psize + 1) << 9; - size_t lsize = ((size_t)bp->lsize + 1) << 9; - - std::vector src(psize); - - vdev->Read(src, psize, addr->offset << 9); - - if(Verify(src, bp->cksum_type, bp->cksum)) - { - if(Decompress(src, dst, lsize, bp->comp_type)) - { - return true; - } - } - } - } - - return false; - } - - bool BlockReader::Verify(std::vector& buff, uint8_t cksum_type, cksum_t& cksum) - { - cksum_t c; - - memset(&c, 0, sizeof(c)); - - switch(cksum_type) - { - case ZIO_CHECKSUM_OFF: - return true; - case ZIO_CHECKSUM_ON: // ??? - case ZIO_CHECKSUM_ZILOG: - case ZIO_CHECKSUM_FLETCHER_2: - fletcher_2_native(buff.data(), buff.size(), &c); - break; - case ZIO_CHECKSUM_ZILOG2: - case ZIO_CHECKSUM_FLETCHER_4: - fletcher_4_native(buff.data(), buff.size(), &c); - break; - case ZIO_CHECKSUM_LABEL: - case ZIO_CHECKSUM_GANG_HEADER: - case ZIO_CHECKSUM_SHA256: - sha256(buff.data(), buff.size(), &c); // TESTME - break; - default: - ASSERT(0); - return false; - } - - ASSERT(cksum == c); - - return cksum == c; - } - - bool BlockReader::Decompress(std::vector& src, std::vector& dst, size_t lsize, uint8_t comp_type) - { - switch(comp_type) - { - case ZIO_COMPRESS_ON: // ??? - case ZIO_COMPRESS_LZJB: - dst.resize(lsize); - lzjb_decompress(src.data(), dst.data(), src.size(), lsize); - break; - case ZIO_COMPRESS_OFF: - case ZIO_COMPRESS_EMPTY: // ??? - dst.swap(src); - break; - case ZIO_COMPRESS_GZIP_1: - case ZIO_COMPRESS_GZIP_2: - case ZIO_COMPRESS_GZIP_3: - case ZIO_COMPRESS_GZIP_4: - case ZIO_COMPRESS_GZIP_5: - case ZIO_COMPRESS_GZIP_6: - case ZIO_COMPRESS_GZIP_7: - case ZIO_COMPRESS_GZIP_8: - case ZIO_COMPRESS_GZIP_9: - gzip_decompress(src.data(), dst.data(), src.size(), lsize); // TESTME - break; - case ZIO_COMPRESS_ZLE: - zle_decompress(src.data(), dst.data(), src.size(), lsize, 64); // TESTME - break; - default: - ASSERT(0); - return false; - } - - return true; - } - - // BlockStream - - BlockStream::BlockStream(Pool* pool, blkptr_t* bp, size_t count) - : BlockReader(pool, bp, count) - { - } - - BlockStream::~BlockStream() - { - } - - bool BlockStream::ReadNext(std::vector& buff) - { - if(m_bpl.empty()) - { - return false; - } - - std::auto_ptr bp(m_bpl.front()); - - m_bpl.pop_front(); - - while(bp->lvl > 0) - { - if(!Read(buff, bp.get())) + if(!FetchBlock(0, block_id, &bp)) { return false; } - Insert((blkptr_t*)buff.data(), buff.size() / sizeof(blkptr_t)); + size_t bytes = 0; - bp = std::auto_ptr(m_bpl.front()); - - m_bpl.pop_front(); - } - - return Read(buff, bp.get()); - } - - bool BlockStream::ReadToEnd(std::vector& dst) - { - dst.clear(); - - // TODO: resize/reserve (how much?) - - size_t i = 0; - - std::vector buff; - - while(ReadNext(buff)) - { - if(i + buff.size() > dst.size()) + if(bp->type != DMU_OT_NONE) { - dst.resize(i + buff.size()); + if(m_cache.id != block_id) + { + if(!m_pool->Read(m_cache.buff, bp)) + { + break; + } + + m_cache.id = block_id; + } + + ASSERT(m_cache.buff.size() == m_datablksize); + + uint8_t* src = m_cache.buff.data() + block_offset; + size_t src_size = m_cache.buff.size() - block_offset; + + bytes = std::min(src_size, size); + + memcpy(ptr, src, bytes); + } + else + { + if(ptr == (uint8_t*)dst && m_node.type == DMU_OT_PLAIN_FILE_CONTENTS) + { + dnode_phys_t* dnode = &m_node; + znode_phys_t* znode = (znode_phys_t*)m_node.bonus(); + + uint8_t* extra = (uint8_t*)(znode + 1); + size_t extra_size = (uint8_t*)(dnode + 1) - extra; + + if(znode->size <= extra_size) + { + bytes = std::min(size, (size_t)znode->size); + + memcpy(ptr, extra, bytes); + + return bytes; + } + } + + size_t src_size = m_datablksize - block_offset; + + bytes = std::min(src_size, size); + + memset(ptr, 0, bytes); } - memcpy(dst.data() + i, buff.data(), buff.size()); + ptr += bytes; + size -= bytes; - i += buff.size(); + block_offset = 0; } + ASSERT(size == 0); + + return ptr - (uint8_t*)dst; + } + + bool BlockReader::FetchBlock(size_t level, uint64_t id, blkptr_t** bp) + { + blklvl_t& lvl = m_tree[level]; + + size_t col_id = (size_t)(id >> (m_node.indblkshift - 7)); + + if(lvl[col_id] == NULL) + { + blkptr_t* bp = NULL; + + if(!FetchBlock(level + 1, col_id, &bp)) + { + return false; + } + + std::vector buff; + + if(bp->type != DMU_OT_NONE) + { + if(!m_pool->Read(buff, bp) || buff.size() != m_indblksize) + { + return false; + } + } + else + { + // FIXME: there may empty pointers in the middle of other valid pointers (???) + + buff.resize(m_indblksize); + + memset(buff.data(), 0, m_indblksize); + } + + blkcol_t* col = new blkcol_t(m_indblkcount); + + memcpy(col->data(), buff.data(), buff.size()); // TODO: read into col->data() directly + + lvl[col_id] = col; + } + + blkcol_t* col = lvl[col_id]; + + uint64_t mask = (1ull << (m_node.indblkshift - 7)) - 1; + + *bp = &col->at((size_t)(id & mask)); + return true; } - - // BlockFile - - BlockFile::BlockFile(Pool* pool, blkptr_t* bp, size_t count) - : BlockReader(pool, bp, count) - , m_psize(0) - , m_lsize(0) - { - std::list l; - - while(!m_bpl.empty()) - { - std::auto_ptr bp(m_bpl.front()); - - m_bpl.pop_front(); - - while(bp->lvl > 0) - { - std::vector buff; - - if(!__super::Read(buff, bp.get())) - { - ASSERT(0); - - return; - } - - Insert((blkptr_t*)buff.data(), buff.size() / sizeof(blkptr_t)); - - bp = std::auto_ptr(m_bpl.front()); - - m_bpl.pop_front(); - } - - m_psize += bp.get()->psize + 1; - m_lsize += bp.get()->lsize + 1; - - l.push_back(bp.release()); - } - - m_psize <<= 9; - m_lsize <<= 9; - - m_bpl.swap(l); - - m_cache.bp = NULL; - } - - BlockFile::~BlockFile() - { - } - - bool BlockFile::Read(void* dst, size_t size, uint64_t offset) - { - // TODO: faster than O(n) access to m_bpl - - uint64_t end = offset + size; - - if(end > m_lsize) - { - return false; - } - - uint8_t* p = (uint8_t*)dst; - - size_t lsize; - uint64_t lstart = 0; - uint64_t lnext; - - for(auto i = m_bpl.begin(); i != m_bpl.end(); i++) - { - blkptr_t* bp = *i; - - lsize = ((size_t)bp->lsize + 1) << 9; - lnext = lstart + lsize; - - if(offset < lnext) - { - if(m_cache.bp != bp) - { - if(!m_pool->Read(m_cache.buff, bp, 1)) - { - return false; - } - - m_cache.bp = bp; - } - - size_t n = std::min(size, (size_t)(lnext - offset)); - - memcpy(p, m_cache.buff.data() + (offset - lstart), n); - - p += n; - size -= n; - - while(size > 0 && ++i != m_bpl.end()) - { - bp = *i; - - lsize = ((size_t)bp->lsize + 1) << 9; - lnext = lstart + lsize; - - if(m_cache.bp != bp) - { - if(!__super::Read(m_cache.buff, bp)) - { - return false; - } - - m_cache.bp = bp; - } - - size_t n = std::min(size, lsize); - - memcpy(p, m_cache.buff.data(), n); - - p += n; - size -= n; - - lstart = lnext; - } - - ASSERT(size == 0); - - return size == 0; - } - - lstart = lnext; - } - - return false; - } } diff --git a/zfs-win/BlockReader.h b/zfs-win/BlockReader.h index 0ddca11..810abb6 100644 --- a/zfs-win/BlockReader.h +++ b/zfs-win/BlockReader.h @@ -29,43 +29,27 @@ namespace ZFS { class BlockReader { - protected: Pool* m_pool; - std::list m_bpl; + dnode_phys_t m_node; + size_t m_datablksize; + size_t m_indblksize; + size_t m_indblkcount; + uint64_t m_size; + struct {uint64_t id; std::vector buff;} m_cache; - void Insert(blkptr_t* bp, size_t count); - bool Read(std::vector& buff, blkptr_t* bp); + typedef std::vector blkcol_t; + typedef std::vector blklvl_t; + typedef std::vector blktree_t; + + blktree_t m_tree; + + bool FetchBlock(size_t level, uint64_t id, blkptr_t** bp); public: - BlockReader(Pool* pool, blkptr_t* bp, size_t count); + BlockReader(Pool* pool, dnode_phys_t* dn); virtual ~BlockReader(); - static bool Verify(std::vector& buff, uint8_t cksum_type, cksum_t& cksum); - static bool Decompress(std::vector& src, std::vector& dst, size_t lsize, uint8_t comp_type); - }; - - class BlockStream : public BlockReader - { - public: - BlockStream(Pool* pool, blkptr_t* bp, size_t count); - virtual ~BlockStream(); - - bool ReadNext(std::vector& buff); - bool ReadToEnd(std::vector& buff); - }; - - class BlockFile : public BlockReader - { - uint64_t m_psize; - uint64_t m_lsize; - struct {blkptr_t* bp; std::vector buff;} m_cache; - - public: - BlockFile(Pool* pool, blkptr_t* bp, size_t count); - virtual ~BlockFile(); - - bool Read(void* dst, size_t size, uint64_t offset); - - uint64_t GetLogicalSize() {return m_lsize;} + size_t Read(void* dst, size_t size, uint64_t offset); + uint64_t GetDataSize() const {return m_size;} }; } \ No newline at end of file diff --git a/zfs-win/DataSet.cpp b/zfs-win/DataSet.cpp index fc8ff41..8218702 100644 --- a/zfs-win/DataSet.cpp +++ b/zfs-win/DataSet.cpp @@ -72,7 +72,7 @@ namespace ZFS { m_head = new ObjectSet(m_pool); - if(!m_head->Init(&m_dataset.bp, 1)) + if(!m_head->Init(&m_dataset.bp)) { return false; } @@ -82,7 +82,7 @@ namespace ZFS { ZFS::ZapObject zap(m_pool); - if(zap.Init(dn.blkptr, dn.nblkptr)) + if(zap.Init(&dn)) { zap.Lookup("mountpoint", m_mountpoint); } @@ -92,7 +92,7 @@ namespace ZFS { ZFS::ZapObject zap(m_pool); - if(zap.Init(dn.blkptr, dn.nblkptr)) + if(zap.Init(&dn)) { for(auto i = zap.begin(); i != zap.end(); i++) { @@ -143,11 +143,11 @@ namespace ZFS } } - bool DataSet::Init(blkptr_t* bp, size_t count) + bool DataSet::Init(blkptr_t* bp) { ObjectSet os(m_pool); - if(!os.Init(bp, count)) + if(!os.Init(bp)) { return false; } @@ -175,6 +175,50 @@ namespace ZFS } } + bool DataSet::Find(const wchar_t* path, DataSet** ds) + { + std::wstring s(path); + + DataSet* tmp[2] = {this, NULL}; + + if(!s.empty()) + { + std::list sl; + + Util::Explode(s, sl, L"/"); + + while(!sl.empty()) + { + std::string s = Util::UTF16To8(sl.front().c_str()); + + sl.pop_front(); + + tmp[1] = NULL; + + for(auto i = tmp[0]->m_children.begin(); i != tmp[0]->m_children.end(); i++) + { + if((*i)->m_name == s) + { + tmp[1] = *i; + + break; + } + } + + if(tmp[1] == NULL) + { + return false; + } + + tmp[0] = tmp[1]; + } + } + + *ds = tmp[0]; + + return true; + } + bool DataSet::Find(const wchar_t* path, dnode_phys_t& dn) { if(m_head == NULL) @@ -217,13 +261,11 @@ namespace ZFS std::wstring dir = s.substr(i, j - i); - wprintf(L"%d-%d %s\n", i, j, dir.c_str()); - i = j != std::string::npos ? j + 1 : std::string::npos; ZFS::ZapObject zap(m_pool); - if(!zap.Init(dn.blkptr, dn.nblkptr)) + if(!zap.Init(&dn)) { return false; } diff --git a/zfs-win/DataSet.h b/zfs-win/DataSet.h index 09d4630..07ddc66 100644 --- a/zfs-win/DataSet.h +++ b/zfs-win/DataSet.h @@ -35,8 +35,8 @@ namespace ZFS void SetDefaults(DataSet* parent); public: - dsl_dir_phys_t m_dir; // TODO: store its properties instead - dsl_dataset_phys_t m_dataset; // TODO: store its properties instead + dsl_dir_phys_t m_dir; + dsl_dataset_phys_t m_dataset; std::string m_name; std::string m_mountpoint; std::list m_children; @@ -46,8 +46,9 @@ namespace ZFS DataSet(Pool* pool); virtual ~DataSet(); - bool Init(blkptr_t* bp, size_t count); + bool Init(blkptr_t* bp); void GetMountPoints(std::list& mpl); + bool Find(const wchar_t* path, DataSet** ds); bool Find(const wchar_t* path, dnode_phys_t& dn); }; } \ No newline at end of file diff --git a/zfs-win/Device.cpp b/zfs-win/Device.cpp index 778b195..8953017 100644 --- a/zfs-win/Device.cpp +++ b/zfs-win/Device.cpp @@ -82,9 +82,9 @@ namespace ZFS if(vdev.dev != NULL) { - dev->Seek(offset + 0x400000); + vdev.dev->Seek(offset + 0x400000); - if(dev->Read(buff.data(), size) == size) + if(vdev.dev->Read(buff.data(), size) == size) { return true; } @@ -113,16 +113,20 @@ namespace ZFS { VirtualDevice& vdev = children[(size_t)rm.m_col[i].devidx]; - // TODO: reconstruct data if vdev.dev is missing or Read fails + bool success = false; if(vdev.dev != NULL) { vdev.dev->Seek(rm.m_col[i].offset + 0x400000); - if(vdev.dev->Read(p, rm.m_col[i].size) != rm.m_col[i].size) - { - return false; - } + success = vdev.dev->Read(p, rm.m_col[i].size) == rm.m_col[i].size; + } + + if(!success) + { + // TODO: reconstruct data + + return false; } p += rm.m_col[i].size; @@ -331,7 +335,7 @@ namespace ZFS if(SetFilePointerEx(m_handle, li, &li2, FILE_CURRENT)) { - printf("%I64d - %I64d (%I64d)\n", li2.QuadPart, li2.QuadPart + size, size); + printf("%I64d - %I64d (%I64d) (%I64d)\n", li2.QuadPart, li2.QuadPart + size, size, m_bytes); } } diff --git a/zfs-win/ObjectSet.cpp b/zfs-win/ObjectSet.cpp index 65b4f3e..87222e3 100644 --- a/zfs-win/ObjectSet.cpp +++ b/zfs-win/ObjectSet.cpp @@ -27,34 +27,31 @@ namespace ZFS ObjectSet::ObjectSet(Pool* pool) : m_pool(pool) , m_objdir(pool) - , m_dnode_reader(NULL) - , m_dnode_count(0) + , m_reader(NULL) + , m_count(0) { } ObjectSet::~ObjectSet() { - delete m_dnode_reader; + delete m_reader; } - bool ObjectSet::Init(blkptr_t* bp, size_t count) + bool ObjectSet::Init(blkptr_t* bp) { ASSERT(bp->type == DMU_OT_OBJSET); { + delete m_reader; + m_objset.clear(); - - if(m_dnode_reader != NULL) - { - delete m_dnode_reader; - - m_dnode_reader = NULL; - } - - m_dnode_count = 0; + m_reader = NULL; + m_count = 0; } - if(!m_pool->Read(m_objset, bp, count)) + ASSERT(bp->lvl == 0); // must not be indirect + + if(!m_pool->Read(m_objset, bp)) { return false; } @@ -66,9 +63,9 @@ namespace ZFS return false; } - m_dnode_reader = new BlockFile(m_pool, os->meta_dnode.blkptr, os->meta_dnode.nblkptr); + m_reader = new BlockReader(m_pool, &os->meta_dnode); - m_dnode_count = (size_t)(m_dnode_reader->GetLogicalSize() / sizeof(dnode_phys_t)); + m_count = (size_t)(m_reader->GetDataSize() / sizeof(dnode_phys_t)); if(os->type == DMU_OST_META || os->type == DMU_OST_ZFS) { @@ -79,7 +76,7 @@ namespace ZFS return false; } - if(!m_objdir.Init(dn.blkptr, dn.nblkptr)) + if(!m_objdir.Init(&dn)) { return false; } @@ -90,12 +87,14 @@ namespace ZFS bool ObjectSet::Read(size_t index, dnode_phys_t* dn, dmu_object_type type) { - if(index >= m_dnode_count || dn == NULL) + if(index >= m_count || dn == NULL) { return false; } - if(!m_dnode_reader->Read(dn, sizeof(dnode_phys_t), (uint64_t)index * sizeof(dnode_phys_t))) + size_t size = sizeof(dnode_phys_t); + + if(m_reader->Read(dn, size, (uint64_t)index * sizeof(dnode_phys_t)) != size) { return false; } diff --git a/zfs-win/ObjectSet.h b/zfs-win/ObjectSet.h index 3a4eeb2..ee507a1 100644 --- a/zfs-win/ObjectSet.h +++ b/zfs-win/ObjectSet.h @@ -32,22 +32,18 @@ namespace ZFS Pool* m_pool; std::vector m_objset; ZapObject m_objdir; - BlockFile* m_dnode_reader; - size_t m_dnode_count; + BlockReader* m_reader; + size_t m_count; public: ObjectSet(Pool* pool); virtual ~ObjectSet(); - bool Init(blkptr_t* bp, size_t count); - - objset_phys_t* operator -> () {return (objset_phys_t*)m_objset.data();} - - // node access - - size_t GetCount() {return m_dnode_count;} - + bool Init(blkptr_t* bp); + size_t GetCount() {return m_count;} bool Read(size_t index, dnode_phys_t* dn, dmu_object_type type = DMU_OT_NONE); bool Read(const char* name, dnode_phys_t* dn, dmu_object_type type = DMU_OT_NONE); + + objset_phys_t* operator -> () {return (objset_phys_t*)m_objset.data();} }; } diff --git a/zfs-win/Pool.cpp b/zfs-win/Pool.cpp index 2960f2d..55ee887 100644 --- a/zfs-win/Pool.cpp +++ b/zfs-win/Pool.cpp @@ -21,7 +21,9 @@ #include "stdafx.h" #include "Pool.h" -#include "BlockReader.h" +#include "Hash.h" +#include "Compress.h" +#include "String.h" namespace ZFS { @@ -35,13 +37,13 @@ namespace ZFS Close(); } - bool Pool::Open(const std::list& paths, const char* name) + bool Pool::Open(const std::list& paths, const wchar_t* name) { Close(); if(name != NULL) { - m_name = name; + m_name = Util::UTF16To8(name); } for(auto i = paths.begin(); i != paths.end(); i++) @@ -90,6 +92,8 @@ namespace ZFS vdev->GetLeaves(leaves); + int missing = 0; + for(auto j = leaves.begin(); j != leaves.end(); j++) { VirtualDevice* leaf = *j; @@ -110,9 +114,25 @@ namespace ZFS if(leaf->dev == NULL) { - return false; + missing++; } } + + if(vdev->type == "raidz" && missing <= vdev->nparity + || vdev->type == "mirror" && missing < vdev->children.size() + || missing == 0) + { + if(missing > 0) + { + wprintf(L"WARNING: vdev %I64d has %d missing disk(s)\n", vdev->id, missing); + } + } + else + { + wprintf(L"ERROR: vdev %I64d has too many (%d) missing disk(s)\n", vdev->id, missing); + + return false; + } } return true; @@ -131,10 +151,117 @@ namespace ZFS m_vdevs.clear(); } - bool Pool::Read(std::vector& buff, blkptr_t* bp, size_t count) + bool Pool::Read(std::vector& dst, blkptr_t* bp) { - BlockStream r(this, bp, count); + for(int i = 0; i < 3; i++) + { + dva_t* addr = &bp->blk_dva[i]; - return r.ReadToEnd(buff); + ASSERT(addr->gang == 0); // TODO: zio_gbh_phys_t (not used ??? never encountered one, yet) + + for(auto i = m_vdevs.begin(); i != m_vdevs.end(); i++) + { + VirtualDevice* vdev = *i; + + if(vdev->id != addr->vdev) + { + continue; + } + + // << 9 or vdev->ashift? + // + // on-disk spec says: "All sizes are stored as the number of 512 byte sectors (minus one) needed to + // represent the size of this block.", but is it still true or outdated? + + size_t psize = ((size_t)bp->psize + 1) << 9; + size_t lsize = ((size_t)bp->lsize + 1) << 9; + + std::vector src(psize); + + if(!vdev->Read(src, psize, addr->offset << 9)) + { + continue; + } + + if(Verify(src, bp->cksum_type, bp->cksum)) + { + if(Decompress(src, dst, lsize, bp->comp_type)) + { + return true; + } + } + } + } + + return false; + } + + bool Pool::Verify(std::vector& buff, uint8_t cksum_type, cksum_t& cksum) + { + cksum_t c; + + memset(&c, 0, sizeof(c)); + + switch(cksum_type) + { + case ZIO_CHECKSUM_OFF: + return true; + case ZIO_CHECKSUM_ON: // ??? + case ZIO_CHECKSUM_ZILOG: + case ZIO_CHECKSUM_FLETCHER_2: + fletcher_2_native(buff.data(), buff.size(), &c); + break; + case ZIO_CHECKSUM_ZILOG2: + case ZIO_CHECKSUM_FLETCHER_4: + fletcher_4_native(buff.data(), buff.size(), &c); + break; + case ZIO_CHECKSUM_LABEL: + case ZIO_CHECKSUM_GANG_HEADER: + case ZIO_CHECKSUM_SHA256: + sha256(buff.data(), buff.size(), &c); // TESTME + break; + default: + ASSERT(0); + return false; + } + + ASSERT(cksum == c); + + return cksum == c; + } + + bool Pool::Decompress(std::vector& src, std::vector& dst, size_t lsize, uint8_t comp_type) + { + switch(comp_type) + { + case ZIO_COMPRESS_ON: // ??? + case ZIO_COMPRESS_LZJB: + dst.resize(lsize); + lzjb_decompress(src.data(), dst.data(), src.size(), lsize); + break; + case ZIO_COMPRESS_OFF: + case ZIO_COMPRESS_EMPTY: // ??? + dst.swap(src); + break; + case ZIO_COMPRESS_GZIP_1: + case ZIO_COMPRESS_GZIP_2: + case ZIO_COMPRESS_GZIP_3: + case ZIO_COMPRESS_GZIP_4: + case ZIO_COMPRESS_GZIP_5: + case ZIO_COMPRESS_GZIP_6: + case ZIO_COMPRESS_GZIP_7: + case ZIO_COMPRESS_GZIP_8: + case ZIO_COMPRESS_GZIP_9: + gzip_decompress(src.data(), dst.data(), src.size(), lsize); // TESTME + break; + case ZIO_COMPRESS_ZLE: + zle_decompress(src.data(), dst.data(), src.size(), lsize, 64); // TESTME + break; + default: + ASSERT(0); + return false; + } + + return true; } } diff --git a/zfs-win/Pool.h b/zfs-win/Pool.h index 9dc0ef4..7add771 100644 --- a/zfs-win/Pool.h +++ b/zfs-win/Pool.h @@ -34,13 +34,16 @@ namespace ZFS std::vector m_devs; std::vector m_vdevs; + static bool Verify(std::vector& buff, uint8_t cksum_type, cksum_t& cksum); + static bool Decompress(std::vector& src, std::vector& dst, size_t lsize, uint8_t comp_type); + public: Pool(); virtual ~Pool(); - bool Open(const std::list& paths, const char* name = NULL); + bool Open(const std::list& paths, const wchar_t* name = NULL); void Close(); - - bool Read(std::vector& buff, blkptr_t* bp, size_t count); + + bool Read(std::vector& buff, blkptr_t* bp); }; } \ No newline at end of file diff --git a/zfs-win/ZapObject.cpp b/zfs-win/ZapObject.cpp index 03f5191..d0605ae 100644 --- a/zfs-win/ZapObject.cpp +++ b/zfs-win/ZapObject.cpp @@ -21,6 +21,7 @@ #include "StdAfx.h" #include "ZapObject.h" +#include "BlockReader.h" namespace ZFS { @@ -34,13 +35,15 @@ namespace ZFS RemoveAll(); } - bool ZapObject::Init(blkptr_t* bp, size_t count) + bool ZapObject::Init(dnode_phys_t* dn) { RemoveAll(); - std::vector buff; + BlockReader r(m_pool, dn); - if(m_pool->Read(buff, bp, count)) + std::vector buff((size_t)r.GetDataSize()); + + if(r.Read(buff.data(), buff.size(), 0)) { if(buff.size() >= sizeof(uint64_t)) { diff --git a/zfs-win/ZapObject.h b/zfs-win/ZapObject.h index d441dbe..80362e5 100644 --- a/zfs-win/ZapObject.h +++ b/zfs-win/ZapObject.h @@ -38,7 +38,7 @@ namespace ZFS ZapObject(Pool* pool); virtual ~ZapObject(); - bool Init(blkptr_t* bp, size_t count); + bool Init(dnode_phys_t* dn); bool Lookup(const char* name, uint64_t& value); bool Lookup(const char* name, std::string& value); diff --git a/zfs-win/main.cpp b/zfs-win/main.cpp index 023c926..b3cea7e 100644 --- a/zfs-win/main.cpp +++ b/zfs-win/main.cpp @@ -32,13 +32,91 @@ namespace ZFS class Context { public: - Pool pool; - DataSet* root; - DataSet* mounted; - std::wstring name; + Pool m_pool; + DataSet* m_root; + DataSet* m_mounted; + std::wstring m_name; - Context() {root = mounted = NULL;} - ~Context() {delete root;} + Context() {m_root = NULL; m_mounted = NULL;} + ~Context() {delete m_root;} + + bool Init(std::list& paths, const std::wstring& name) + { + m_name = name; + + if(!m_pool.Open(paths, name.c_str())) + { + wprintf(L"Failed to open pool\n"); + + return false; + } + + m_root = new ZFS::DataSet(&m_pool); + + if(!m_root->Init(&m_pool.m_devs.front()->m_active->rootbp)) + { + wprintf(L"Failed to read root dataset\n"); + + return false; + } + + m_mounted = m_root; + + return true; + } + + bool SetDataSet(std::wstring& dataset) + { + if(!m_root->Find(dataset.c_str(), &m_mounted)) + { + std::list sl; + + sl.push_back(UTF8To16(m_pool.m_name.c_str())); + + if(!dataset.empty()) + { + sl.push_back(dataset); + } + + wprintf(L"Cannot find dataset '%s'\n", Implode(sl, L"/").c_str()); + + return false; + } + + return true; + } + + void List(const DataSet* ds, std::wstring parent = L"") + { + std::wstring s; + + if(!parent.empty()) + { + s = parent + L"/"; + } + + s += Util::UTF8To16(ds->m_name.c_str()); + + wprintf(L"%s\n", s.c_str()); + + for(auto i = ds->m_children.begin(); i != ds->m_children.end(); i++) + { + List(*i, s); + } + } + }; + + class FileContext + { + public: + dnode_phys_t node; + BlockReader reader; + + FileContext(Pool* pool, dnode_phys_t* dn) + : node(*dn) + , reader(pool, dn) + { + } }; static void UnixTimeToFileTime(uint64_t t, FILETIME* pft) @@ -68,7 +146,7 @@ namespace ZFS memset(&dn, 0, sizeof(dn)); - if(!ctx->mounted->Find(FileName, dn)) + if(!ctx->m_mounted->Find(FileName, dn)) { dn.type = 0; } @@ -96,7 +174,7 @@ namespace ZFS DokanFileInfo->IsDirectory = 1; } - DokanFileInfo->Context = (ULONG64)new dnode_phys_t(dn); + DokanFileInfo->Context = (ULONG64)new FileContext(&ctx->m_pool, &dn); return CreationDisposition == OPEN_ALWAYS ? ERROR_ALREADY_EXISTS : 0; } @@ -111,12 +189,12 @@ namespace ZFS dnode_phys_t dn; - if(!ctx->mounted->Find(FileName, dn) || dn.type != DMU_OT_DIRECTORY_CONTENTS) + if(!ctx->m_mounted->Find(FileName, dn) || dn.type != DMU_OT_DIRECTORY_CONTENTS) { return -ERROR_PATH_NOT_FOUND; } - DokanFileInfo->Context = (ULONG64)new dnode_phys_t(dn); + DokanFileInfo->Context = (ULONG64)new FileContext(&ctx->m_pool, &dn); return 0; } @@ -142,9 +220,7 @@ namespace ZFS if(DokanFileInfo->Context != 0) { - dnode_phys_t* dn = (dnode_phys_t*)DokanFileInfo->Context; - - delete dn; + delete (FileContext*)DokanFileInfo->Context; DokanFileInfo->Context = 0; } @@ -162,9 +238,7 @@ namespace ZFS if(DokanFileInfo->Context != 0) { - dnode_phys_t* dn = (dnode_phys_t*)DokanFileInfo->Context; - - delete dn; + delete (FileContext*)DokanFileInfo->Context; DokanFileInfo->Context = 0; } @@ -182,9 +256,22 @@ namespace ZFS { Context* ctx = (Context*)DokanFileInfo->DokanOptions->GlobalContext; - wprintf(L"%s: %s\n", __FUNCTIONW__, FileName); + // wprintf(L"%s: %s %d %I64d\n", __FUNCTIONW__, FileName, BufferLength, Offset); - return -ERROR_ACCESS_DENIED; + FileContext* fctx = (FileContext*)DokanFileInfo->Context; + + znode_phys_t* znode = (znode_phys_t*)fctx->node.bonus(); + + BufferLength = (DWORD)std::min(znode->size - Offset, BufferLength); + + *ReadLength = fctx->reader.Read(Buffer, BufferLength, Offset); + + if(*ReadLength != BufferLength) + { + return -ERROR_READ_FAULT; + } + + return 0; } static int WINAPI WriteFile( @@ -222,9 +309,9 @@ namespace ZFS wprintf(L"%s: %s\n", __FUNCTIONW__, FileName); - dnode_phys_t* dn = (dnode_phys_t*)DokanFileInfo->Context; + FileContext* fctx = (FileContext*)DokanFileInfo->Context; - znode_phys_t* node = (znode_phys_t*)dn->bonus(); + znode_phys_t* node = (znode_phys_t*)fctx->node.bonus(); UnixTimeToFileTime(node->crtime[0], &HandleFileInformation->ftCreationTime); UnixTimeToFileTime(node->atime[0], &HandleFileInformation->ftLastAccessTime); @@ -237,9 +324,9 @@ namespace ZFS HandleFileInformation->dwFileAttributes |= FILE_ATTRIBUTE_READONLY; // TODO } - if(dn->type == DMU_OT_DIRECTORY_CONTENTS) + if(fctx->node.type == DMU_OT_DIRECTORY_CONTENTS) { - HandleFileInformation->dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; + HandleFileInformation->dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; // TODO: symlinks pointing to directories } HandleFileInformation->dwVolumeSerialNumber = 0; @@ -276,32 +363,44 @@ namespace ZFS if(DokanFileInfo->Context != 0) { - dnode_phys_t* dn = (dnode_phys_t*)DokanFileInfo->Context; + FileContext* fctx = (FileContext*)DokanFileInfo->Context; - if(dn->type != DMU_OT_DIRECTORY_CONTENTS) + if(fctx->node.type != DMU_OT_DIRECTORY_CONTENTS) { return -1; } - ZFS::ZapObject zap(&ctx->pool); + ZFS::ZapObject zap(&ctx->m_pool); - if(!zap.Init(dn->blkptr, dn->nblkptr)) + if(!zap.Init(&fctx->node)) { return -1; } + bool everything = wcscmp(SearchPattern, L"*") == 0; + bool wildcards = wcschr(SearchPattern, '*') != NULL || wcschr(SearchPattern, '?') != NULL; + for(auto i = zap.begin(); i != zap.end(); i++) { + std::string fn = i->first; + uint64_t index = 0; - if(!zap.Lookup(i->first.c_str(), index)) + if(!zap.Lookup(fn.c_str(), index)) { return false; } + std::wstring wfn = Util::UTF8To16(fn.c_str()); + + if(!everything && !(wildcards ? PathMatchSpec(wfn.c_str(), SearchPattern) : wfn == SearchPattern)) + { + continue; + } + dnode_phys_t subdn; - if(!ctx->mounted->m_head->Read((size_t)ZFS_DIRENT_OBJ(index), &subdn)) + if(!ctx->m_mounted->m_head->Read((size_t)ZFS_DIRENT_OBJ(index), &subdn)) { return false; } @@ -319,7 +418,7 @@ namespace ZFS if(subdn.type == DMU_OT_DIRECTORY_CONTENTS) { - fd.dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; + fd.dwFileAttributes |= FILE_ATTRIBUTE_DIRECTORY; // TODO: symlinks pointing to directories } UnixTimeToFileTime(node->crtime[0], &fd.ftCreationTime); @@ -329,9 +428,7 @@ namespace ZFS fd.nFileSizeLow = (DWORD)(node->size); fd.nFileSizeHigh = (DWORD)(node->size >> 32); - std::wstring fn = Util::UTF8To16(i->first.c_str()); - - wcscpy(fd.cFileName, fn.c_str()); + wcscpy(fd.cFileName, wfn.c_str()); fd.cAlternateFileName[0] = 0; // ??? @@ -468,7 +565,7 @@ namespace ZFS uint64_t total = 0; - for(auto i = ctx->pool.m_vdevs.begin(); i != ctx->pool.m_vdevs.end(); i++) + for(auto i = ctx->m_pool.m_vdevs.begin(); i != ctx->m_pool.m_vdevs.end(); i++) { VirtualDevice* vdev = *i; @@ -503,12 +600,12 @@ namespace ZFS if(TotalNumberOfFreeBytes != NULL) { - *TotalNumberOfFreeBytes = total - ctx->root->m_dir.used_bytes; + *TotalNumberOfFreeBytes = total - ctx->m_root->m_dir.used_bytes; } if(FreeBytesAvailable != NULL) { - *FreeBytesAvailable = total - ctx->root->m_dir.used_bytes; + *FreeBytesAvailable = total - ctx->m_root->m_dir.used_bytes; } return 0; @@ -530,7 +627,7 @@ namespace ZFS if(VolumeNameBuffer != NULL) { - wcscpy(VolumeNameBuffer, ctx->name.c_str()); + wcscpy(VolumeNameBuffer, ctx->m_name.c_str()); } if(VolumeSerialNumber != NULL) @@ -576,15 +673,13 @@ static void usage() printf( "ZFS for Windows\n" "\n" - "commands:\n" - " mount \n" - " list \n" - "\n" - "pool:\n" - " Any combination of devices (\\\\.\\PhysicalDriveN) or files.\n" + "usage:\n" + " mount \n" + " list \n" "\n" "examples:\n" " zfs-win.exe mount m \"rpool/ROOT/opensolaris\" \"\\\\.\\PhysicalDrive1\" \"\\\\.\\PhysicalDrive2\"\n" + " zfs-win.exe list \"Virtual Machine-flat.vmdk\"\n" ); } @@ -594,7 +689,9 @@ int _tmain(int argc, _TCHAR* argv[]) wchar_t drive = 0; std::list paths; - std::list dataset; + std::wstring pool; + std::wstring dataset; + bool list_only = false; if(wcsicmp(argv[1], L"mount") == 0) { @@ -602,7 +699,15 @@ int _tmain(int argc, _TCHAR* argv[]) drive = argv[2][0]; - Util::Explode(std::wstring(argv[3]), dataset, L"/"); + std::list sl; + + Util::Explode(std::wstring(argv[3]), sl, L"/"); + + pool = sl.front(); + + sl.pop_front(); + + dataset = Implode(sl, '/'); for(int i = 4; i < argc; i++) { @@ -618,7 +723,7 @@ int _tmain(int argc, _TCHAR* argv[]) paths.push_back(argv[i]); } - return -1; // TODO + list_only = true; } if(paths.empty()) {usage(); return -1;} @@ -626,69 +731,35 @@ int _tmain(int argc, _TCHAR* argv[]) /* paths.clear(); - paths.push_back(L"\\\\.\\PhysicalDrive1"); paths.push_back(L"\\\\.\\PhysicalDrive2"); paths.push_back(L"\\\\.\\PhysicalDrive3"); - paths.push_back(L"\\\\.\\PhysicalDrive4"); + // paths.push_back(L"\\\\.\\PhysicalDrive4"); - Util::Explode(std::wstring(L"share/backup"), dataset, L"/"); + pool = L"share"; + dataset = L"backup"; */ ZFS::Context ctx; - std::wstring name = dataset.front(); - - dataset.pop_front(); - - if(!ctx.pool.Open(paths, UTF16To8(name.c_str()).c_str())) + if(!ctx.Init(paths, pool)) { - printf("Failed to open pool\n"); - return -1; } - ctx.root = new ZFS::DataSet(&ctx.pool); - - if(!ctx.root->Init(&ctx.pool.m_devs.front()->m_active->rootbp, 1)) + if(list_only) { - printf("Failed to read root dataset\n"); + ctx.List(ctx.m_root); + return 0; + } + + if(!ctx.SetDataSet(dataset)) + { return -1; } - ZFS::DataSet* ds = ctx.root; - - ctx.name = name; - - while(!dataset.empty()) - { - name = dataset.front(); - - dataset.pop_front(); - - ZFS::DataSet* ds2 = NULL; - - for(auto i = ds->m_children.begin(); i != ds->m_children.end(); i++) - { - if((*i)->m_name == UTF16To8(name.c_str())) - { - ds2 = *i; - - break; - } - } - - if(ds2 == NULL) {usage(); return -1;} - - ds = ds2; - - // ctx.name += L"/" + name; - } - - ctx.mounted = ds; - // DOKAN_OPTIONS options;