related tdf#162798: add script to create the list of external packages

Change-Id: Ifc24cb42df76c50f0ca40e3c793f36c355e483d8
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/178730
Tested-by: Xisco Fauli <xiscofauli@libreoffice.org>
Reviewed-by: Xisco Fauli <xiscofauli@libreoffice.org>

external/python3/ExternalPackage_python3.mk

@@ -176,12 +176,10 @@ endif
 # test - probably unnecessary? was explicitly removed #i116738#
 # venv - why would we need virtual environments
 #
-# These lists are now sorted with "LC_COLLATE=C sort", by using
+# Call generateExternalPackage.py to update the lists below
-#   find Lib/ -name "*.py" | sort | sed -e 's/^/\t/' -e 's/$/ \\/'
 #
 
 $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib,\
-	LICENSE \
 	Lib/__future__.py \
 	Lib/__phello__.foo.py \
 	Lib/_aix_support.py \
@@ -413,11 +411,11 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 
 $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/ctypes/macholib,\
 	Lib/ctypes/macholib/README.ctypes \
-	Lib/ctypes/macholib/fetch_macholib \
-	Lib/ctypes/macholib/fetch_macholib.bat \
 	Lib/ctypes/macholib/__init__.py \
 	Lib/ctypes/macholib/dyld.py \
 	Lib/ctypes/macholib/dylib.py \
+	Lib/ctypes/macholib/fetch_macholib \
+	Lib/ctypes/macholib/fetch_macholib.bat \
 	Lib/ctypes/macholib/framework.py \
 ))
 
@@ -687,10 +685,10 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 ))
 
 $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/lib2to3,\
-	Lib/lib2to3/__init__.py \
-	Lib/lib2to3/__main__.py \
 	Lib/lib2to3/Grammar.txt \
 	Lib/lib2to3/PatternGrammar.txt \
+	Lib/lib2to3/__init__.py \
+	Lib/lib2to3/__main__.py \
 	Lib/lib2to3/btm_matcher.py \
 	Lib/lib2to3/btm_utils.py \
 	Lib/lib2to3/fixer_base.py \
@@ -758,7 +756,7 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 	Lib/lib2to3/fixes/fix_zip.py \
 ))
 
-$(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/pgen2,\
+$(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/lib2to3/pgen2,\
 	Lib/lib2to3/pgen2/__init__.py \
 	Lib/lib2to3/pgen2/conv.py \
 	Lib/lib2to3/pgen2/driver.py \
@@ -789,8 +787,6 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 	Lib/multiprocessing/__init__.py \
 	Lib/multiprocessing/connection.py \
 	Lib/multiprocessing/context.py \
-	Lib/multiprocessing/dummy/__init__.py \
-	Lib/multiprocessing/dummy/connection.py \
 	Lib/multiprocessing/forkserver.py \
 	Lib/multiprocessing/heap.py \
 	Lib/multiprocessing/managers.py \
@@ -822,6 +818,10 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 	Lib/pydoc_data/topics.py \
 ))
 
+$(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/site-packages,\
+	Lib/site-packages/README.txt \
+))
+
 $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/unittest,\
 	Lib/unittest/__init__.py \
 	Lib/unittest/__main__.py \
@@ -906,8 +906,4 @@ $(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/p
 	Lib/zoneinfo/_zoneinfo.py \
 ))
 
-$(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/lib/site-packages,\
-	Lib/site-packages/README.txt \
-))
-
 # vim: set noet sw=4 ts=4:

external/python3/generateExternalPackage.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from os import path, walk
+
+if __name__ == '__main__':
+
+    ignoredPackages = ["dbm", "curses", "test", "tkinter", "turtledemo", "sqlite3", "idlelib", "venv"]
+
+    coreDir = path.dirname(path.dirname(path.dirname(path.abspath(__file__))))
+    pythonDir = path.join(coreDir, "workdir/UnpackedTarball/python3")
+    libDir = path.join(pythonDir, "Lib")
+    subDirDict = {}
+    for subdir, dirs, files in walk(libDir):
+        filesList = []
+        relPythonDir = path.relpath(subdir, pythonDir)
+        relLibDir = path.relpath(subdir, libDir)
+
+        isShipped = True
+        for package in ignoredPackages:
+            if relLibDir.startswith(package):
+                isShipped = False
+                continue
+
+        if not isShipped:
+            continue
+
+        if "/test" in relLibDir:
+            continue
+
+        for fileName in sorted(files):
+            if fileName.endswith(".pyc"):
+                continue
+            filesList.append(path.join(relPythonDir, fileName))
+        if filesList:
+            subDirDict[relPythonDir] = filesList
+
+    for k,v in sorted(subDirDict.items()):
+        print()
+        if k == "Lib/msilib":
+            print("ifeq (WNT,$(OS))")
+        print("$(eval $(call gb_ExternalPackage_add_unpacked_files,python3,$(LIBO_BIN_FOLDER)/python-core-$(PYTHON_VERSION)/{},\\".format(k.lower()))
+        for fileName in sorted(v):
+            print("\t{} \\".format(fileName))
+        print("))")
+        if k == "Lib/msilib":
+            print("endif")
+
+# vim: set shiftwidth=4 softtabstop=4 expandtab: