exploring datasets

zfs-win/BlockReader.cpp

@@ -97,18 +97,24 @@ namespace ZFS
 	{
 		if(m_bpl.empty())
 		{
-			for(size_t i = 0; i < count && bp[i].type != DMU_OT_NONE; i++)
+			for(size_t i = 0; i < count; i++)
 			{
-				m_bpl.push_back(new blkptr_t(bp[i]));
+				if(bp[i].type != DMU_OT_NONE)
+				{
+					m_bpl.push_back(new blkptr_t(bp[i]));
+				}
 			}
 		}
 		else
 		{
 			std::list<blkptr_t*> l;
 
-			for(size_t i = 0; i < count && bp[i].type != DMU_OT_NONE; i++)
+			for(size_t i = 0; i < count; i++)
 			{
-				l.push_back(new blkptr_t(bp[i]));
+				if(bp[i].type != DMU_OT_NONE)
+				{
+					l.push_back(new blkptr_t(bp[i]));
+				}
 			}
 
 			m_bpl.insert(m_bpl.begin(), l.begin(), l.end());

zfs-win/Device.h

@@ -46,7 +46,7 @@ namespace ZFS
 		uint64_t is_log;
 		std::vector<VirtualDevice> children;
 
-		void Init(NameValueList* nvl) throw(...);
+		void Init(NameValueList* nvl);
 		bool Read(std::vector<uint8_t>& buff, uint64_t size, uint64_t offset);
 		VirtualDevice* Find(uint64_t guid_to_find);
 		void GetLeaves(std::list<VirtualDevice*>& leaves);

zfs-win/ObjectSet.cpp

@@ -1,4 +1,25 @@
-#include "StdAfx.h"
+/* 
+ *	Copyright (C) 2010 Gabest
+ *	http://code.google.com/p/zfs-win/
+ *
+ *  This Program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2, or (at your option)
+ *  any later version.
+ *   
+ *  This Program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *   
+ *  You should have received a copy of the GNU General Public License
+ *  along with GNU Make; see the file COPYING.  If not, write to
+ *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 
+ *  http://www.gnu.org/copyleft/gpl.html
+ *
+ */
+
+#include "stdafx.h"
 #include "ObjectSet.h"
 
 namespace ZFS

zfs-win/ObjectSet.h

@@ -1,3 +1,24 @@
+/* 
+ *	Copyright (C) 2010 Gabest
+ *	http://code.google.com/p/zfs-win/
+ *
+ *  This Program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2, or (at your option)
+ *  any later version.
+ *   
+ *  This Program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ *   
+ *  You should have received a copy of the GNU General Public License
+ *  along with GNU Make; see the file COPYING.  If not, write to
+ *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 
+ *  http://www.gnu.org/copyleft/gpl.html
+ *
+ */
+
 #pragma once
 
 #include "Pool.h"

zfs-win/ZapObject.cpp

@@ -81,8 +81,25 @@ namespace ZFS
 
 		if(i != end())
 		{
-			value = std::string((char*)i->second->data(), i->second->size());
+			std::vector<uint8_t>* buff = i->second;
-			
+
+			if(!buff->empty())
+			{
+				char* ptr = (char*)buff->data();
+				size_t size = buff->size();
+
+				while(size > 0 && ptr[size - 1] == 0) 
+				{
+					size--;
+				}
+
+				value = std::string(ptr, size);
+			}
+			else
+			{
+				value.empty();
+			}
+
 			return true;
 		}
 
@@ -130,49 +147,66 @@ namespace ZFS
 
 	void ZapObject::ParseFat(std::vector<uint8_t>& buff)
 	{
-		size_t half_size = buff.size() / 2; // first half wasted ???
+		// NOTE: not sure about this, the documentation is outdated, 0x4000 granularity seems to work
+
+		ASSERT(buff.size() >= 0x8000 && (buff.size() & 0x3fff) == 0);
+
+		if(buff.size() < 0x8000) 
+		{
+			return;
+		}
 
 		zap_phys_t* zap = (zap_phys_t*)buff.data(); // TODO: zap->ptrtbl ???
-		zap_leaf_phys_t* leaf = (zap_leaf_phys_t*)(buff.data() + half_size);
 
-		zap_leaf_entry_t* e = (zap_leaf_entry_t*)(uint8_t*)&leaf->hash[half_size / 32];
+		size_t n = buff.size() / 0x4000 - 1;
-		zap_leaf_entry_t* e_end = (zap_leaf_entry_t*)(buff.data() + buff.size());
+		size_t m = 0x4000 - offsetof(zap_leaf_phys_t, hash[0x4000 / 32]);
 
-		for(size_t i = 0, n = e_end - e; i < n; i++)
+		uint8_t* ptr = buff.data() + 0x4000;
+
+		for(size_t i = 0; i < n; i++, ptr += 0x4000)
 		{
-			if(e[i].type != ZAP_CHUNK_ENTRY)
+			zap_leaf_phys_t* leaf = (zap_leaf_phys_t*)ptr;
+
+			ASSERT(leaf->block_type == ZBT_LEAF);
+
+			zap_leaf_entry_t* e = (zap_leaf_entry_t*)&leaf->hash[0x4000 / 32];
+
+			for(size_t i = 0, n = m / sizeof(zap_leaf_entry_t); i < n; i++)
 			{
-				continue;
+				if(e[i].type != ZAP_CHUNK_ENTRY)
-			}
+				{
+					continue;
+				}
 
-			std::vector<uint8_t> name(e[i].name_numints);
+				std::vector<uint8_t> name(e[i].name_numints);
 
-			if(!ParseArray(name, e, e[i].name_chunk) || name.empty())
+				if(!ParseArray(name, e, e[i].name_chunk) || name.empty())
-			{
+				{
-				continue;
+					continue;
-			}
+				}
 
-			std::vector<uint8_t>* value = new std::vector<uint8_t>(e[i].value_numints * e[i].value_intlen);
+				std::vector<uint8_t>* value = new std::vector<uint8_t>(e[i].value_numints * e[i].value_intlen);
 
-			if(!ParseArray(*value, e, e[i].value_chunk))
+				if(!ParseArray(*value, e, e[i].value_chunk))
-			{
+				{
-				delete value;
+					delete value;
 
-				continue;
+					continue;
-			}
+				}
 
-			std::string s((char*)name.data(), name.size() - 1);
+				std::string s((char*)name.data(), name.size() - 1);
 
-			auto j = find(s);
+				auto j = find(s);
 
-			if(j != end())
+				if(j != end())
-			{
+				{
-				delete j->second;
+					delete j->second;
 
-				erase(j);
+					erase(j);
-			}
+				}
 			
-			(*this)[s] = value;
+				(*this)[s] = value;
+			}
 		}
 	}
 

zfs-win/ZapObject.h

@@ -25,7 +25,7 @@
 
 namespace ZFS
 {
-	class ZapObject : protected std::map<std::string, std::vector<uint8_t>*>
+	class ZapObject : public std::map<std::string, std::vector<uint8_t>*>
 	{
 		void RemoveAll();
 		void ParseMicro(std::vector<uint8_t>& buff);

zfs-win/main.cpp

@@ -21,9 +21,8 @@
 
 #include "stdafx.h"
 #include "Pool.h"
-#include "Device.h"
+#include "DataSet.h"
-#include "ObjectSet.h"
+#include <time.h>
-#include "ZapObject.h"
 
 int _tmain(int argc, _TCHAR* argv[])
 {
@@ -75,48 +74,36 @@ int _tmain(int argc, _TCHAR* argv[])
 
 	if(os.Read(p, &dev->m_active->rootbp, 1))
 	{
-		ASSERT(os.count() > 2);
+		ZFS::DataSet ds;
-
+		
-		dnode_phys_t* root_dataset = os["root_dataset"];
+		if(ds.Read(p, os))
-
-		dnode_phys_t* head_dataset = NULL;
-
-		if(root_dataset != NULL && root_dataset->type == DMU_OT_DSL_DIR)
 		{
-			dsl_dir_phys_t* dir = (dsl_dir_phys_t*)root_dataset->bonus;
+			std::list<ZFS::DataSet*> mpl;
 
-			size_t index = (size_t)dir->head_dataset_obj;
+			ds.GetMountPoints(mpl);
 
-			if(index < os.count())
+			for(auto i = mpl.begin(); i != mpl.end(); i++)
 			{
-				head_dataset = os[index];
+				if((*i)->m_mountpoint != "/") continue;
-			}
-		}
 
-		if(head_dataset != NULL && head_dataset->type == DMU_OT_DSL_DATASET)
-		{
-			dsl_dataset_phys_t* ds = (dsl_dataset_phys_t*)head_dataset->bonus;
-
-			if(ds->bp.type == DMU_OT_OBJSET)
-			{
 				ZFS::ObjectSet os;
 
-				if(os.Read(p, &ds->bp, 1))
+				clock_t t = clock();
+
+				if(os.Read(p, &(*i)->m_dataset->bp, 1)) // <-- this can be huge (all files and directories), compression helps a bit
 				{
+					printf("%d %d\n", clock() - t, os.count());
+
 					dnode_phys_t* root = os["ROOT"];
 
 					if(root != NULL && root->type == DMU_OT_DIRECTORY_CONTENTS)
 					{
-						znode_phys_t* node = (znode_phys_t*)root->bonus;
+						znode_phys_t* node = (znode_phys_t*)root->bonus();
 
-						std::vector<uint8_t> buff;
+						ZFS::ZapObject zap;
 
-						if(p.Read(buff, root->blkptr, root->nblkptr))
+						if(zap.Read(p, root->blkptr, root->nblkptr))
 						{
-							mzap_phys_t* mzap = (mzap_phys_t*)buff.data();
-
-							// finally, arrived at the root directory
-
 							int i = 0;
 						}
 					}

zfs-win/zfs-win.vcxproj

@@ -65,6 +65,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="BlockReader.h" />
+    <ClInclude Include="DataSet.h" />
     <ClInclude Include="Device.h" />
     <ClInclude Include="Compress.h" />
     <ClInclude Include="Hash.h" />
@@ -78,6 +79,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="BlockReader.cpp" />
+    <ClCompile Include="DataSet.cpp" />
     <ClCompile Include="Device.cpp" />
     <ClCompile Include="Compress.cpp" />
     <ClCompile Include="NameValueList.cpp" />

zfs-win/zfs-win.vcxproj.filters

@@ -48,6 +48,9 @@
     <ClInclude Include="ObjectSet.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="DataSet.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="stdafx.cpp">
@@ -80,5 +83,8 @@
     <ClCompile Include="ObjectSet.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="DataSet.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

zfs-win/zfs.h

@@ -226,10 +226,6 @@ struct uberblock_t
 
 //
 
-#define	OBJSET_PHYS_SIZE 2048
-#define	OBJSET_OLD_PHYS_SIZE 1024
-#define	OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL<<0)
-
 #define	DNODE_SHIFT 9 /* 512 bytes */
 #define	DN_MIN_INDBLKSHIFT 10 /* 1k */
 #define	DN_MAX_INDBLKSHIFT 14 /* 16k */
@@ -250,11 +246,11 @@ enum dmu_object_type
 	DMU_OT_OBJECT_DIRECTORY, /* ZAP */
 	DMU_OT_OBJECT_ARRAY, /* UINT64 */
 	DMU_OT_PACKED_NVLIST, /* UINT8 (XDR by nvlist_pack/unpack) */
-	DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */
+	DMU_OT_PACKED_NVLIST_SIZE, /* UINT64 */ /* bonus = uint64_t */
 	DMU_OT_BPLIST, /* UINT64 */
 	DMU_OT_BPLIST_HDR, /* UINT64 */
 	/* spa: */
-	DMU_OT_SPACE_MAP_HEADER, /* UINT64 */
+	DMU_OT_SPACE_MAP_HEADER, /* UINT64 */ /* bonus = space_map_obj_t */
 	DMU_OT_SPACE_MAP = 8, /* UINT64 */
 	/* zil: */
 	DMU_OT_INTENT_LOG, /* UINT64 */
@@ -266,12 +262,12 @@ enum dmu_object_type
 	DMU_OT_DSL_DIR_CHILD_MAP, /* ZAP */
 	DMU_OT_DSL_DS_SNAP_MAP, /* ZAP */
 	DMU_OT_DSL_PROPS, /* ZAP */
-	DMU_OT_DSL_DATASET = 16, /* UINT64 */
+	DMU_OT_DSL_DATASET = 16, /* UINT64 */ /* bonus = dsl_dataset_phys_t */
 	/* zpl: */
-	DMU_OT_ZNODE, /* ZNODE */
+	DMU_OT_ZNODE, /* ZNODE */ /* bonus = znode_phys_t */
 	DMU_OT_OLDACL, /* Old ACL */
 	DMU_OT_PLAIN_FILE_CONTENTS, /* UINT8 */
-	DMU_OT_DIRECTORY_CONTENTS, /* ZAP */
+	DMU_OT_DIRECTORY_CONTENTS, /* ZAP */ /* bonus = znode_phys_t */
 	DMU_OT_MASTER_NODE, /* ZAP */
 	DMU_OT_UNLINKED_SET, /* ZAP */
 	/* zvol: */
@@ -312,6 +308,9 @@ enum dmu_objset_type
 	DMU_OST_NUMTYPES
 };
 
+#define	ZIL_REPLAY_NEEDED 0x1 /* replay needed - internal only */
+#define	ZIL_CLAIM_LR_SEQ_VALID 0x2 /* zh_claim_lr_seq field is valid */
+
 struct zil_header_t
 {
 	uint64_t claim_txg; /* txg in which log blocks were claimed */
@@ -323,6 +322,16 @@ struct zil_header_t
 	uint64_t pad[3];
 };
 
+struct zil_chain_t
+{
+	uint64_t pad;
+	blkptr_t next_blk; /* next block in chain */
+	uint64_t nused; /* bytes in log block used */
+	zio_eck_t eck; /* block trailer */
+};
+
+// TODO: rest of the zil structs
+
 struct dnode_phys_t
 {
 	uint8_t type; /* dmu_object_type_t */
@@ -336,17 +345,19 @@ struct dnode_phys_t
 	uint16_t datablkszsec; /* data block size in 512b sectors */
 	uint16_t bonuslen; /* length of dn_bonus */
 	uint8_t pad2[4];
-
-	/* accounting is protected by dn_dirty_mtx */
 	uint64_t maxblkid; /* largest allocated block ID */
 	uint64_t used; /* bytes (or sectors) of disk space */
-
 	uint64_t pad3[4];
+	blkptr_t blkptr[1]; // 1-3 elements
+	uint8_t pad4[DN_MAX_BONUSLEN];
 
-	blkptr_t blkptr[1];
+	uint8_t* bonus() {return (uint8_t*)&blkptr[nblkptr];}
-	uint8_t bonus[DN_MAX_BONUSLEN];
 };
 
+#define	OBJSET_PHYS_SIZE 2048
+#define	OBJSET_OLD_PHYS_SIZE 1024
+#define	OBJSET_FLAG_USERACCOUNTING_COMPLETE (1ULL<<0)
+
 struct objset_phys_t
 {
 	union