Merge pull request #82 from jonnor/pip-install-github-actions

Support installing with pip, and run tests with GitHub Actions

Author: Zepan

.github/workflows/tests.yaml

@@ -0,0 +1,36 @@
+name: Python package
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install OS dependencies
+        run: |
+          sudo apt-get install -yqq python-tk git
+      - name: Install Python dependencies
+        run: |
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements.dev.txt ]; then pip install -r requirements.dev.txt; fi
+      - name: Install as pip package
+        run: |
+          pip install ./ -v
+      - name: Test installed package
+        working-directory: examples/auto_test
+        run: |
+          python auto_test.py
+

README.md

@@ -162,6 +162,27 @@ make
 ./mbnet
 ```
 
+
+## How to install
+
+TinyMaix can be installed as a Python package
+
+```
+pip install git+https://github.com/sipeed/TinyMaix@master
+```
+
+TinyMaix requires [Tensorflow](https://www.tensorflow.org/) version `<= 2.14`.
+There are multiple options for how to install this, see the TensorFlow documentation.
+
+For example:
+```
+pip install 'tensorflow-cpu<=2.14.1'
+```
+
+NOTE: Tensorflow 2.14 supports up until Python 3.11.
+However, *Python 3.12 is not supported* by Tensorflow 2.14.
+
+
 ## How to use (API)
 ### Load Model
 tm_err_t tm_load  (tm_mdl_t* mdl, const uint8_t* bin, uint8_t*buf, tm_cb_t cb, tm_mat_t* in);   
@@ -199,17 +220,17 @@ And now just put them into your project, compile it~
 
 ## How to train/convert models
 There are training scripts in examples/mnist to learn how to train simple mnist models. 
-> Note: you need install TensorFlow (>=2.7) first.
+> Note: you need install TinyMaix first.
 
 After training and save h5 models, you can use scripts in tools to convert to tmdl or c header files.   
 
 1. h5_to_tflite.py   
   convert h5 model to float or int8 quant tflite files   
-  python3 h5_to_tflite.py h5/mnist.h5 tflite/mnist_f.tflite 0   
-  python3 h5_to_tflite.py h5/mnist.h5 tflite/mnist_q.tflite 1 quant_img_mnist/ 0to1   
+  python -m tinymaix.tools.h5_to_tflite h5/mnist.h5 tflite/mnist_f.tflite 0
+  python -m tinymaix.tools.h5_to_tflite h5/mnist.h5 tflite/mnist_q.tflite 1 quant_img_mnist/ 0to1
 2. tflite2tmdl.py  
   convert tflite file to tmdl or c header files.    
-  python3 tflite2tmdl.py tflite/mnist_q.tflite tmdl/mnist_q.tmdl int8 1 28,28,1 10  
+  python -m tinymaix.tools.tflite2tmdl tflite/mnist_q.tflite tmdl/mnist_q.tmdl int8 1 28,28,1 10
 ```
 ================ pack model head ================
 mdl_type   =0

README_ZH.md

@@ -198,17 +198,17 @@ TinyMaix的核心文件只有这5个：tm_model.c, tm_layers.c, tinymaix.h, tm_p
 
 ## 怎样训练/转换模型
 在examples/mnist下有训练脚本可以学习如何训练基础的mnist模型
-> 注意：你需要先安装TensorFlow (>=2.7) 环境.
+> 注意：你需要先安装TinyMaix 环境.
 
 完成训练并保存h5模型后，你可以使用以下脚本转换原始模型到tmdl或者c头文件。 
 
 1. h5_to_tflite.py   
   转换h5模型到浮点或者int8量化的tflite模型
-  python3 h5_to_tflite.py h5/mnist.h5 tflite/mnist_f.tflite 0   
-  python3 h5_to_tflite.py h5/mnist.h5 tflite/mnist_q.tflite 1 quant_img_mnist/ 0to1   
+  python -m tinymaix.tools.h5_to_tflite h5/mnist.h5 tflite/mnist_f.tflite 0
+  python -m tinymaix.tools.h5_to_tflite h5/mnist.h5 tflite/mnist_q.tflite 1 quant_img_mnist/ 0to1
 2. tflite2tmdl.py  
   转换tflite文件到tmdl或者c头文件   
-  python3 tflite2tmdl.py tflite/mnist_q.tflite tmdl/mnist_q.tmdl int8 1 28,28,1 10  
+  python -m tinymaix.tools.tflite2tmdl tflite/mnist_q.tflite tmdl/mnist_q.tmdl int8 1 28,28,1 10
 ```
 ================ pack model head ================
 mdl_type   =0

examples/auto_test/auto_test.py

@@ -1,19 +1,35 @@
 import os,sys,time
-from subprocess import *
-import datetime,time
-import threading
+import datetime
+import subprocess
+
 
 ### This script do auto testing before release code.
 test_list = ["mnist_f", "mnist_q", "mbnet_f", "mbnet_q"] 
 
 
-def runcmd(cmd):
-    r=Popen(cmd,stdin=PIPE,stdout=PIPE,stderr=PIPE, shell=True)
+def runcmd(cmd, cwd=None):
+
+    stdout = subprocess.check_output(cmd, shell=True, cwd=cwd)
+    # stdin=PIPE,stdout=PIPE,stderr=PIPE
     a=[]
-    for line in r.stdout.readlines():
-        a.append(line.decode("utf8").strip())
+    for line in stdout.decode("utf8").strip().split('\n'):
+        a.append(line.strip())
     return a
 
+def run_h5_to_tflite(cmd, **kwargs):
+    program = 'python -m tinymaix.tools.h5_to_tflite'
+    cmd = program + ' ' + cmd
+    return runcmd(cmd, **kwargs)
+
+def run_tflite2tmdl(cmd, **kwargs):
+    program = 'python -m tinymaix.tools.tflite2tmdl'
+    cmd = program + ' ' + cmd
+    return runcmd(cmd, **kwargs)
+
+tools_dir = '../../tools/'
+mnist_dir = '../mnist'
+mbnet_dir = '../mbnet'
+
 
 print("This script only test INT8/FP32, you need change OPT0&OPT1")
 t00= time.time()
@@ -22,25 +38,20 @@ def runcmd(cmd):
     t0= time.time()
     print("========Step1.1: test MNIST fp32")
     print("====Step1.1.1: MNIST fp32 cvt")
-    cmd="cd ../../tools/ && python3 h5_to_tflite.py h5/mnist_valid.h5 tflite/mnist_valid_f.tflite 0 && python3 tflite2tmdl.py tflite/mnist_valid_f.tflite tmdl/mnist_valid_f.tmdl fp32 1 28,28,1 10"
-    res = runcmd(cmd)
-    print(res[-1])
-    if res[-1] == "Saved to tinymaix model header to tmdl/mnist_valid_f.h":
-        print("====Step1.1.1: OK~")
-    else:
-        print("====Step1.1.1: ERR!!!")
-        exit(-1)
+    run_h5_to_tflite("h5/mnist_valid.h5 tflite/mnist_valid_f.tflite 0", cwd=tools_dir)
+    res = run_tflite2tmdl("tflite/mnist_valid_f.tflite tmdl/mnist_valid_f.tmdl fp32 1 28,28,1 10", cwd=tools_dir)
+    assert 'Saved to tmdl/mnist_valid_f.tmdl' in res[-1], ('Step 1.1.1 ERR', res)
+
 
     print("====Step1.1.2: MNIST fp32 compile&run")
-    cmd="cd ../mnist && sed -i 's/#define TM_MDL_TYPE     TM_MDL_INT8/#define TM_MDL_TYPE     TM_MDL_FP32/g' ../../include/tm_port.h && rm -rf build && mkdir build && cd build && cmake .. && make && ./mnist"
-    res = runcmd(cmd)
+    cmd="sed -i 's/#define TM_MDL_TYPE     TM_MDL_INT8/#define TM_MDL_TYPE     TM_MDL_FP32/g' ../../include/tm_port.h"
+    res = runcmd(cmd, cwd=mnist_dir)
+    res = runcmd("rm -rf build && mkdir build && cd build && cmake .. && make && ./mnist", cwd=mnist_dir)
     print(res[-1])
-    if res[-1] == "### Predict output is: Number 2, prob 1.000":
-        print("====Step1.1.2: OK~")
-        runcmd("rm ../../tools/tmdl/mnist_valid_f.tmdl")  #clean tmdl
-    else:
-        print("====Step1.1.2: ERR!!!")
-        exit(-2)
+    assert 'Predict output is: Number 2, prob 1.0' in res[-1], res[-1]
+
+    runcmd("rm ../../tools/tmdl/mnist_valid_f.tmdl")  #clean tmdl
+
     t1= time.time()
     print("========Step1.1: test MNIST fp32 OK~  use %.1fs"%(t1-t0))
 
@@ -49,26 +60,17 @@ def runcmd(cmd):
     t0= time.time()
     print("========Step1.2: test MNIST int8")
     print("====Step1.2.1: MNIST int8 cvt")
-    cmd="cd ../../tools/ && python3 h5_to_tflite.py h5/mnist_valid.h5 tflite/mnist_valid_q.tflite 1 quant_img_mnist/ 0to1 && python3 tflite2tmdl.py tflite/mnist_valid_q.tflite tmdl/mnist_valid_q.tmdl int8 1 28,28,1 10"
-    res = runcmd(cmd)
-    print(res[-1])
-    if res[-1] == "Saved to tinymaix model header to tmdl/mnist_valid_q.h":
-        print("====Step1.2.1: OK~")
-    else:
-        print("====Step1.2.1: ERR!!!")
-        exit(-1)
+    res = run_h5_to_tflite("h5/mnist_valid.h5 tflite/mnist_valid_q.tflite 1 quant_img_mnist/ 0to1", cwd=tools_dir)
+    res = run_tflite2tmdl("tflite/mnist_valid_q.tflite tmdl/mnist_valid_q.tmdl int8 1 28,28,1 10", cwd=tools_dir)
 
     print("====Step1.2.2: MNIST int8 compile&run")
-
-    cmd="cd ../mnist && sed -i 's/#define TM_MDL_TYPE     TM_MDL_FP32/#define TM_MDL_TYPE     TM_MDL_INT8/g' ../../include/tm_port.h && rm -rf build && mkdir build && cd build && cmake .. && make && ./mnist"
-    res = runcmd(cmd)
+    cmd="sed -i 's/#define TM_MDL_TYPE     TM_MDL_FP32/#define TM_MDL_TYPE     TM_MDL_INT8/g' ../../include/tm_port.h"
+    res = runcmd(cmd, cwd=mnist_dir)
+    res = runcmd("rm -rf build && mkdir build && cd build && cmake .. && make && ./mnist", cwd=mnist_dir)
     print(res[-1])
-    if res[-1] == "### Predict output is: Number 2, prob 0.996":
-        print("====Step1.2.2: OK~")
-        runcmd("rm ../../tools/tmdl/mnist_valid_q.tmdl")  #clean tmdl
-    else:
-        print("====Step1.2.2: ERR!!!")
-        exit(-2)
+    assert 'Predict output is: Number 2, prob 0.996' in res[-1], res[-1]
+
+    runcmd("rm ../../tools/tmdl/mnist_valid_q.tmdl")  #clean tmdl
 
     t1= time.time()
     print("========Step1.2: test MNIST int8 OK~  use %.1fs"%(t1-t0))
@@ -79,27 +81,17 @@ def runcmd(cmd):
     t0= time.time()
     print("========Step2.1: test MBNET fp32")
     print("====Step2.1.1: MBNET fp32 cvt")
-    cmd="cd ../../tools/ && python3 h5_to_tflite.py h5/mbnet128_0.25.h5 tflite/mbnet128_0.25_f.tflite 0 && python3 tflite2tmdl.py tflite/mbnet128_0.25_f.tflite tmdl/mbnet128_0.25_f.tmdl fp32 1 128,128,3 1000"
-    res = runcmd(cmd)
-    print(res[-1])
-    if res[-1] == "Saved to tinymaix model header to tmdl/mbnet128_0.25_f.h":
-        print("====Step2.1.1: OK~")
-    else:
-        print("====Step2.1.1: ERR!!!")
-        exit(-1)
+    res = run_h5_to_tflite("h5/mbnet128_0.25.h5 tflite/mbnet128_0.25_f.tflite 0", cwd=tools_dir)
+    res = run_tflite2tmdl("tflite/mbnet128_0.25_f.tflite tmdl/mbnet128_0.25_f.tmdl fp32 1 128,128,3 1000", cwd=tools_dir)
 
     print("====Step2.1.2: MBNET fp32 compile&run")
-
-    cmd="cd ../mbnet && sed -i 's/#define TM_MDL_TYPE     TM_MDL_INT8/#define TM_MDL_TYPE     TM_MDL_FP32/g' ../../include/tm_port.h && rm -rf build && mkdir build && cd build && cmake .. && make && ./mbnet"
-    res = runcmd(cmd)
+    cmd="sed -i 's/#define TM_MDL_TYPE     TM_MDL_INT8/#define TM_MDL_TYPE     TM_MDL_FP32/g' ../../include/tm_port.h"
+    res = runcmd(cmd, cwd=mbnet_dir)
+    res = runcmd("rm -rf build && mkdir build && cd build && cmake .. && make && ./mbnet", cwd=mbnet_dir)
     print(res[-1])
-    if res[-1] == "### Predict output is: Class 292 (tiger, Panthera tigris), Prob 0.866" or \
-        res[-1] == "### Predict output is: Class 292 (tiger, Panthera tigris), Prob 0.891":
-        print("====Step2.1.2: OK~")
-        runcmd("rm ../../tools/tmdl/mbnet128_0.25_f.tmdl")  #clean tmdl
-    else:
-        print("====Step2.1.2: ERR!!!")
-        exit(-2)
+    assert 'Class 292 (tiger, Panthera tigris), Prob 0.8' in res[-1], res[-1]
+
+    runcmd("rm ../../tools/tmdl/mbnet128_0.25_f.tmdl")  #clean tmdl
 
     t1= time.time()
     print("========Step2.1: test MBNET fp32 OK~  use %.1fs"%(t1-t0))
@@ -109,26 +101,17 @@ def runcmd(cmd):
     t0= time.time()
     print("========Step2.2: test MBNET int8")
     print("====Step2.2.1: MBNET int8 cvt")
-    cmd="cd ../../tools/ && python3 h5_to_tflite.py h5/mbnet128_0.25.h5 tflite/mbnet128_0.25_q.tflite 1 quant_img128/ 0to1 && python3 tflite2tmdl.py tflite/mbnet128_0.25_q.tflite tmdl/mbnet128_0.25_q.tmdl int8 1 128,128,3 1000"
-    res = runcmd(cmd)
-    print(res[-1])
-    if res[-1] == "Saved to tinymaix model header to tmdl/mbnet128_0.25_q.h":
-        print("====Step2.2.1: OK~")
-    else:
-        print("====Step2.2.1: ERR!!!")
-        exit(-1)
+    res = run_h5_to_tflite("h5/mbnet128_0.25.h5 tflite/mbnet128_0.25_q.tflite 1 quant_img128/ 0to1", cwd=tools_dir)
+    res = run_tflite2tmdl("tflite/mbnet128_0.25_q.tflite tmdl/mbnet128_0.25_q.tmdl int8 1 128,128,3 1000", cwd=tools_dir)
 
     print("====Step2.2.2: MBNET int8 compile&run")
-
-    cmd="cd ../mbnet && sed -i 's/#define TM_MDL_TYPE     TM_MDL_FP32/#define TM_MDL_TYPE     TM_MDL_INT8/g' ../../include/tm_port.h && rm -rf build && mkdir build && cd build && cmake .. && make && ./mbnet"
-    res = runcmd(cmd)
+    cmd="sed -i 's/#define TM_MDL_TYPE     TM_MDL_FP32/#define TM_MDL_TYPE     TM_MDL_INT8/g' ../../include/tm_port.h"
+    res = runcmd(cmd, cwd=mbnet_dir)
+    res = runcmd("rm -rf build && mkdir build && cd build && cmake .. && make && ./mbnet", cwd=mbnet_dir)
     print(res[-1])
-    if res[-1] == "### Predict output is: Class 292 (tiger, Panthera tigris), Prob 0.824":
-        print("====Step2.2.2: OK~")
-        runcmd("rm ../../tools/tmdl/mbnet128_0.25_q.tmdl")  #clean tmdl
-    else:
-        print("====Step2.2.2: ERR!!!")
-        exit(-2)
+    assert 'Class 292 (tiger, Panthera tigris), Prob 0.8' in res[-1]
+
+    runcmd("rm ../../tools/tmdl/mbnet128_0.25_q.tmdl")  #clean tmdl
 
     t1= time.time()
     print("========Step2.2: test MBNET int8 OK~  use %.1fs"%(t1-t0))

examples/auto_test/requirements.txt

@@ -25,7 +25,7 @@ limitations under the License.
     #elif TM_MDL_TYPE==TM_MDL_FP16
     #include "../../tools/tmdl/mbnet128_0.25_fp16.h"
     #elif TM_MDL_TYPE==TM_MDL_INT8
-    #include "../../tools/mbtestv2.h"
+    #include "../../tools/tmdl/mbnet128_0.25_q.h"
     #elif TM_MDL_TYPE==TM_MDL_FP8_143
     #include "../../tools/tmdl/mbnet128_0.25_fp8_143.h"
     #elif TM_MDL_TYPE==TM_MDL_FP8_152

examples/mbnet/main.c

@@ -0,0 +1,35 @@
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src", "include", "tools" ]
+
+[tool.hatch.build.targets.wheel.sources]
+"src" = "tinymaix/src"
+"include" = "tinymaix/include"
+"tools" = "tinymaix/tools"
+
+[project]
+name = "TinyMaix"
+readme = "README.md"
+requires-python = ">= 3.8"
+keywords = ["machine learning", "deep learning", "microcontroller", "embedded"]
+version = "0.0.1"
+
+dependencies = [
+  "keras<3.0.0",
+  "pillow>=10.0.0",
+]
+
+classifiers = [
+  "License :: OSI Approved :: Apache Software License",
+]
+
+
+[project.urls]
+Homepage = "https://github.com/sipeed/TinyMaix"
+Documentation = "https://github.com/sipeed/TinyMaix"
+Repository = "https://github.com/sipeed/TinyMaix"
+

pyproject.toml

@@ -0,0 +1,2 @@
+tensorflow-cpu==2.14.1
+numpy

requirements.dev.txt

@@ -18,11 +18,7 @@
 import tensorflow as tf
 import time, struct
 from PIL import Image
-try:
-    from .tflite_reader import read_tflite
-except:
-    from tflite_reader import read_tflite
-
+from .tflite_reader import read_tflite
 
 # constant
 TM_MDL_INT8  = 0