From e5a6bc76cb1e4f9655be31520b4e2898f0e3a024 Mon Sep 17 00:00:00 2001 From: gabest11 Date: Fri, 23 Jul 2010 08:29:29 +0000 Subject: [PATCH] exploring datasets --- zfs-win/BlockReader.cpp | 14 +++-- zfs-win/Device.h | 2 +- zfs-win/ObjectSet.cpp | 23 ++++++++- zfs-win/ObjectSet.h | 21 ++++++++ zfs-win/ZapObject.cpp | 92 ++++++++++++++++++++++----------- zfs-win/ZapObject.h | 2 +- zfs-win/main.cpp | 47 ++++++----------- zfs-win/zfs-win.vcxproj | 2 + zfs-win/zfs-win.vcxproj.filters | 6 +++ zfs-win/zfs.h | 39 +++++++++----- 10 files changed, 168 insertions(+), 80 deletions(-) diff --git a/zfs-win/BlockReader.cpp b/zfs-win/BlockReader.cpp index e3023b0..f4bc17c 100644 --- a/zfs-win/BlockReader.cpp +++ b/zfs-win/BlockReader.cpp @@ -97,18 +97,24 @@ namespace ZFS { if(m_bpl.empty()) { - for(size_t i = 0; i < count && bp[i].type != DMU_OT_NONE; i++) + for(size_t i = 0; i < count; i++) { - m_bpl.push_back(new blkptr_t(bp[i])); + if(bp[i].type != DMU_OT_NONE) + { + m_bpl.push_back(new blkptr_t(bp[i])); + } } } else { std::list l; - for(size_t i = 0; i < count && bp[i].type != DMU_OT_NONE; i++) + for(size_t i = 0; i < count; i++) { - l.push_back(new blkptr_t(bp[i])); + if(bp[i].type != DMU_OT_NONE) + { + l.push_back(new blkptr_t(bp[i])); + } } m_bpl.insert(m_bpl.begin(), l.begin(), l.end()); diff --git a/zfs-win/Device.h b/zfs-win/Device.h index 511d64d..06505a5 100644 --- a/zfs-win/Device.h +++ b/zfs-win/Device.h @@ -46,7 +46,7 @@ namespace ZFS uint64_t is_log; std::vector children; - void Init(NameValueList* nvl) throw(...); + void Init(NameValueList* nvl); bool Read(std::vector& buff, uint64_t size, uint64_t offset); VirtualDevice* Find(uint64_t guid_to_find); void GetLeaves(std::list& leaves); diff --git a/zfs-win/ObjectSet.cpp b/zfs-win/ObjectSet.cpp index 326bdf6..8147908 100644 --- a/zfs-win/ObjectSet.cpp +++ b/zfs-win/ObjectSet.cpp @@ -1,4 +1,25 @@ -#include "StdAfx.h" +/* + * Copyright (C) 2010 Gabest + * http://code.google.com/p/zfs-win/ + * + * This Program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This Program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GNU Make; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * http://www.gnu.org/copyleft/gpl.html + * + */ + +#include "stdafx.h" #include "ObjectSet.h" namespace ZFS diff --git a/zfs-win/ObjectSet.h b/zfs-win/ObjectSet.h index a6f3eff..71dd668 100644 --- a/zfs-win/ObjectSet.h +++ b/zfs-win/ObjectSet.h @@ -1,3 +1,24 @@ +/* + * Copyright (C) 2010 Gabest + * http://code.google.com/p/zfs-win/ + * + * This Program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This Program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GNU Make; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * http://www.gnu.org/copyleft/gpl.html + * + */ + #pragma once #include "Pool.h" diff --git a/zfs-win/ZapObject.cpp b/zfs-win/ZapObject.cpp index e2cab3a..b876c2a 100644 --- a/zfs-win/ZapObject.cpp +++ b/zfs-win/ZapObject.cpp @@ -81,8 +81,25 @@ namespace ZFS if(i != end()) { - value = std::string((char*)i->second->data(), i->second->size()); - + std::vector* buff = i->second; + + if(!buff->empty()) + { + char* ptr = (char*)buff->data(); + size_t size = buff->size(); + + while(size > 0 && ptr[size - 1] == 0) + { + size--; + } + + value = std::string(ptr, size); + } + else + { + value.empty(); + } + return true; } @@ -130,49 +147,66 @@ namespace ZFS void ZapObject::ParseFat(std::vector& buff) { - size_t half_size = buff.size() / 2; // first half wasted ??? + // NOTE: not sure about this, the documentation is outdated, 0x4000 granularity seems to work + + ASSERT(buff.size() >= 0x8000 && (buff.size() & 0x3fff) == 0); + + if(buff.size() < 0x8000) + { + return; + } zap_phys_t* zap = (zap_phys_t*)buff.data(); // TODO: zap->ptrtbl ??? - zap_leaf_phys_t* leaf = (zap_leaf_phys_t*)(buff.data() + half_size); - zap_leaf_entry_t* e = (zap_leaf_entry_t*)(uint8_t*)&leaf->hash[half_size / 32]; - zap_leaf_entry_t* e_end = (zap_leaf_entry_t*)(buff.data() + buff.size()); + size_t n = buff.size() / 0x4000 - 1; + size_t m = 0x4000 - offsetof(zap_leaf_phys_t, hash[0x4000 / 32]); - for(size_t i = 0, n = e_end - e; i < n; i++) + uint8_t* ptr = buff.data() + 0x4000; + + for(size_t i = 0; i < n; i++, ptr += 0x4000) { - if(e[i].type != ZAP_CHUNK_ENTRY) + zap_leaf_phys_t* leaf = (zap_leaf_phys_t*)ptr; + + ASSERT(leaf->block_type == ZBT_LEAF); + + zap_leaf_entry_t* e = (zap_leaf_entry_t*)&leaf->hash[0x4000 / 32]; + + for(size_t i = 0, n = m / sizeof(zap_leaf_entry_t); i < n; i++) { - continue; - } + if(e[i].type != ZAP_CHUNK_ENTRY) + { + continue; + } - std::vector name(e[i].name_numints); + std::vector name(e[i].name_numints); - if(!ParseArray(name, e, e[i].name_chunk) || name.empty()) - { - continue; - } + if(!ParseArray(name, e, e[i].name_chunk) || name.empty()) + { + continue; + } - std::vector* value = new std::vector(e[i].value_numints * e[i].value_intlen); + std::vector* value = new std::vector(e[i].value_numints * e[i].value_intlen); - if(!ParseArray(*value, e, e[i].value_chunk)) - { - delete value; + if(!ParseArray(*value, e, e[i].value_chunk)) + { + delete value; - continue; - } + continue; + } - std::string s((char*)name.data(), name.size() - 1); + std::string s((char*)name.data(), name.size() - 1); - auto j = find(s); + auto j = find(s); - if(j != end()) - { - delete j->second; + if(j != end()) + { + delete j->second; - erase(j); - } + erase(j); + } - (*this)[s] = value; + (*this)[s] = value; + } } } diff --git a/zfs-win/ZapObject.h b/zfs-win/ZapObject.h index c9b1348..2fd1385 100644 --- a/zfs-win/ZapObject.h +++ b/zfs-win/ZapObject.h @@ -25,7 +25,7 @@ namespace ZFS { - class ZapObject : protected std::map*> + class ZapObject : public std::map*> { void RemoveAll(); void ParseMicro(std::vector& buff); diff --git a/zfs-win/main.cpp b/zfs-win/main.cpp index 877005d..2ebd2dd 100644 --- a/zfs-win/main.cpp +++ b/zfs-win/main.cpp @@ -21,9 +21,8 @@ #include "stdafx.h" #include "Pool.h" -#include "Device.h" -#include "ObjectSet.h" -#include "ZapObject.h" +#include "DataSet.h" +#include int _tmain(int argc, _TCHAR* argv[]) { @@ -75,48 +74,36 @@ int _tmain(int argc, _TCHAR* argv[]) if(os.Read(p, &dev->m_active->rootbp, 1)) { - ASSERT(os.count() > 2); - - dnode_phys_t* root_dataset = os["root_dataset"]; - - dnode_phys_t* head_dataset = NULL; - - if(root_dataset != NULL && root_dataset->type == DMU_OT_DSL_DIR) + ZFS::DataSet ds; + + if(ds.Read(p, os)) { - dsl_dir_phys_t* dir = (dsl_dir_phys_t*)root_dataset->bonus; + std::list mpl; - size_t index = (size_t)dir->head_dataset_obj; + ds.GetMountPoints(mpl); - if(index < os.count()) + for(auto i = mpl.begin(); i != mpl.end(); i++) { - head_dataset = os[index]; - } - } + if((*i)->m_mountpoint != "/") continue; - if(head_dataset != NULL && head_dataset->type == DMU_OT_DSL_DATASET) - { - dsl_dataset_phys_t* ds = (dsl_dataset_phys_t*)head_dataset->bonus; - - if(ds->bp.type == DMU_OT_OBJSET) - { ZFS::ObjectSet os; - if(os.Read(p, &ds->bp, 1)) + clock_t t = clock(); + + if(os.Read(p, &(*i)->m_dataset->bp, 1)) // <-- this can be huge (all files and directories), compression helps a bit { + printf("%d %d\n", clock() - t, os.count()); + dnode_phys_t* root = os["ROOT"]; if(root != NULL && root->type == DMU_OT_DIRECTORY_CONTENTS) { - znode_phys_t* node = (znode_phys_t*)root->bonus; + znode_phys_t* node = (znode_phys_t*)root->bonus(); - std::vector buff; + ZFS::ZapObject zap; - if(p.Read(buff, root->blkptr, root->nblkptr)) + if(zap.Read(p, root->blkptr, root->nblkptr)) { - mzap_phys_t* mzap = (mzap_phys_t*)buff.data(); - - // finally, arrived at the root directory - int i = 0; } } diff --git a/zfs-win/zfs-win.vcxproj b/zfs-win/zfs-win.vcxproj index fde2746..c6e5eaa 100644 --- a/zfs-win/zfs-win.vcxproj +++ b/zfs-win/zfs-win.vcxproj @@ -65,6 +65,7 @@ + @@ -78,6 +79,7 @@ + diff --git a/zfs-win/zfs-win.vcxproj.filters b/zfs-win/zfs-win.vcxproj.filters index a8cf2c8..782e0de 100644 --- a/zfs-win/zfs-win.vcxproj.filters +++ b/zfs-win/zfs-win.vcxproj.filters @@ -48,6 +48,9 @@ Header Files + + Header Files + @@ -80,5 +83,8 @@ Source Files + + Source Files + \ No newline at end of file diff --git a/zfs-win/zfs.h b/zfs-win/zfs.h index 6bf5c76..ac830bb 100644 --- a/zfs-win/zfs.h +++ b/zfs-win/zfs.h @@ -226,10 +226,6 @@ struct uberblock_t // -#define OBJSET_PHYS_SIZE 2048 -#define OBJSET_OLD_PHYS_SIZE 1024 -#define OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL<<0) - #define DNODE_SHIFT 9 /* 512 bytes */ #define DN_MIN_INDBLKSHIFT 10 /* 1k */ #define DN_MAX_INDBLKSHIFT 14 /* 16k */ @@ -250,11 +246,11 @@ enum dmu_object_type DMU_OT_OBJECT_DIRECTORY, /* ZAP */ DMU_OT_OBJECT_ARRAY, /* UINT64 */ DMU_OT_PACKED_NVLIST, /* UINT8 (XDR by nvlist_pack/unpack) */ - DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */ + DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */ /* bonus = uint64_t */ DMU_OT_BPLIST, /* UINT64 */ DMU_OT_BPLIST_HDR, /* UINT64 */ /* spa: */ - DMU_OT_SPACE_MAP_HEADER, /* UINT64 */ + DMU_OT_SPACE_MAP_HEADER, /* UINT64 */ /* bonus = space_map_obj_t */ DMU_OT_SPACE_MAP = 8, /* UINT64 */ /* zil: */ DMU_OT_INTENT_LOG, /* UINT64 */ @@ -266,12 +262,12 @@ enum dmu_object_type DMU_OT_DSL_DIR_CHILD_MAP, /* ZAP */ DMU_OT_DSL_DS_SNAP_MAP, /* ZAP */ DMU_OT_DSL_PROPS, /* ZAP */ - DMU_OT_DSL_DATASET = 16, /* UINT64 */ + DMU_OT_DSL_DATASET = 16, /* UINT64 */ /* bonus = dsl_dataset_phys_t */ /* zpl: */ - DMU_OT_ZNODE, /* ZNODE */ + DMU_OT_ZNODE, /* ZNODE */ /* bonus = znode_phys_t */ DMU_OT_OLDACL, /* Old ACL */ DMU_OT_PLAIN_FILE_CONTENTS, /* UINT8 */ - DMU_OT_DIRECTORY_CONTENTS, /* ZAP */ + DMU_OT_DIRECTORY_CONTENTS, /* ZAP */ /* bonus = znode_phys_t */ DMU_OT_MASTER_NODE, /* ZAP */ DMU_OT_UNLINKED_SET, /* ZAP */ /* zvol: */ @@ -312,6 +308,9 @@ enum dmu_objset_type DMU_OST_NUMTYPES }; +#define ZIL_REPLAY_NEEDED 0x1 /* replay needed - internal only */ +#define ZIL_CLAIM_LR_SEQ_VALID 0x2 /* zh_claim_lr_seq field is valid */ + struct zil_header_t { uint64_t claim_txg; /* txg in which log blocks were claimed */ @@ -323,6 +322,16 @@ struct zil_header_t uint64_t pad[3]; }; +struct zil_chain_t +{ + uint64_t pad; + blkptr_t next_blk; /* next block in chain */ + uint64_t nused; /* bytes in log block used */ + zio_eck_t eck; /* block trailer */ +}; + +// TODO: rest of the zil structs + struct dnode_phys_t { uint8_t type; /* dmu_object_type_t */ @@ -336,17 +345,19 @@ struct dnode_phys_t uint16_t datablkszsec; /* data block size in 512b sectors */ uint16_t bonuslen; /* length of dn_bonus */ uint8_t pad2[4]; - - /* accounting is protected by dn_dirty_mtx */ uint64_t maxblkid; /* largest allocated block ID */ uint64_t used; /* bytes (or sectors) of disk space */ - uint64_t pad3[4]; + blkptr_t blkptr[1]; // 1-3 elements + uint8_t pad4[DN_MAX_BONUSLEN]; - blkptr_t blkptr[1]; - uint8_t bonus[DN_MAX_BONUSLEN]; + uint8_t* bonus() {return (uint8_t*)&blkptr[nblkptr];} }; +#define OBJSET_PHYS_SIZE 2048 +#define OBJSET_OLD_PHYS_SIZE 1024 +#define OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL<<0) + struct objset_phys_t { union