穿插编译Python-2.7.13到ARM(aarch32)—— 支持sqlite3

作者:彭东林

邮箱:pengdonglin137@163.com

QQ: 405728433

 

环境

主机: ubuntu14.04
64bit

开发板: qemu +
vexpress-a9
(参考: http://www.cnblogs.com/pengdonglin137/p/6442583.html)

 

工具链: arm-none-linux-gnueabi-gcc  (gcc version 4.8.3
20140320)

Python版本: Python-2.7.13

 

概述

前一篇博文(交叉编译Python-2.7.13到ARM(aarch32)平台)介绍了移植python到aarch32上面,但是发现产生广大模块都不可知因此,可以在板子上面执行下的命令测试一下:

 1 [root@vexpress ]# python /usr/lib/python2.7/test/test___all__.py
 2 Traceback (most recent call last):
 3   File "/usr/lib/python2.7/test/test___all__.py", line 3, in <module>
 4     import unittest
 5   File "/usr/lib/python2.7/unittest/__init__.py", line 58, in <module>
 6     from .result import TestResult
 7   File "/usr/lib/python2.7/unittest/result.py", line 9, in <module>
 8     from . import util
 9   File "/usr/lib/python2.7/unittest/util.py", line 2, in <module>
10     from collections import namedtuple, OrderedDict
11   File "/usr/lib/python2.7/collections.py", line 20, in <module>
12     from _collections import deque, defaultdict
13 ImportError: No module named _collections

好望此间找不交_collections模块。

对比x86_64底编译结果:

1 ls x86_64/build/lib.linux-x86_64-2.7/
2 array.so*       _codecs_hk.so*       cPickle.so*       _curses_panel.so*  future_builtins.so*  itertools.so*      mmap.so*              parser.so*    _socket.so*   _sysconfigdata.py   time.so*
3 audioop.so*     _codecs_iso2022.so*  crypt.so*         _curses.so*        grp.so*              _json.so*          _multibytecodec.so*   pyexpat.so*   spwd.so*      _sysconfigdata.pyc  unicodedata.so*
4 binascii.so*    _codecs_jp.so*       cStringIO.so*     datetime.so*       _hashlib.so*         linuxaudiodev.so*  _multiprocessing.so*  _random.so*   _sqlite3.so*  _sysconfigdata.pyo  zlib.so*
5 _bisect.so*     _codecs_kr.so*       _csv.so*          _elementtree.so*   _heapq.so*           _locale.so*        nis.so*               readline.so*  _ssl.so*      syslog.so*
6 cmath.so*       _codecs_tw.so*       _ctypes.so*       fcntl.so*          _hotshot.so*         _lsprof.so*        operator.so*          resource.so*  strop.so*     termios.so*
7 _codecs_cn.so*  _collections.so*     _ctypes_test.so*  _functools.so*     _io.so*              math.so*           ossaudiodev.so*       select.so*    _struct.so*   _testcapi.so*

而aarch32的编译结果:

1 $ls aarch32/build/lib.linux2-arm-2.7/
2 audioop.so*     _codecs_iso2022.so*  _codecs_tw.so*  _ctypes.so*       _elementtree.so*     _json.so*          mmap.so*              nis.so*      resource.so*        termios.so*
3 _codecs_cn.so*  _codecs_jp.so*       crypt.so*       _ctypes_test.so*  future_builtins.so*  linuxaudiodev.so*  _multibytecodec.so*   parser.so*   _sysconfigdata.py   _testcapi.so*
4 _codecs_hk.so*  _codecs_kr.so*       _csv.so*        datetime.so*      _hotshot.so*         _lsprof.so*        _multiprocessing.so*  pyexpat.so*  _sysconfigdata.pyc

得看到,aarch32上面缺少了不少仓房,
比如_collections.so,将来这些库会被装置到/usr/lib/python2.7/lib-dynload下面,
所以下面要说的即使是拿少的这些库将回去!

正文

1、通过分析setup.py发现问题

在函数build_extensions中恰恰起self.extensions中存放的是索要编译库,
通过以加以打印:

 1 diff --git a/setup.py b/setup.py
 2 index 54054c2..bc16bb1 100644
 3 --- a/setup.py
 4 +++ b/setup.py
 5 @@ -178,6 +178,7 @@ class PyBuildExt(build_ext):
 6  
 7      def build_extensions(self):
 8  
 9 +        print "build_extensions enter."
10          # Detect which modules should be compiled
11          missing = self.detect_modules()
12  
13 @@ -191,6 +192,9 @@ class PyBuildExt(build_ext):
14              extensions.append(ctypes)
15          self.extensions = extensions
16  
17 +        for ext in self.extensions:
18 +            print "extensions: ", ext.name
19 +
20          # Fix up the autodetected modules, prefixing all the source files
21          # with Modules/ and adding Python's include directory to the path.
22          (srcdir,) = sysconfig.get_config_vars('srcdir')
23 @@ -217,6 +221,8 @@ class PyBuildExt(build_ext):
24          # Python header files
25          headers = [sysconfig.get_config_h_filename()]
26          headers += glob(os.path.join(sysconfig.get_path('include'), "*.h"))
27 +
28 +        print "builtin_module_names: ", sys.builtin_module_names
29          for ext in self.extensions[:]:
30              ext.sources = [ find_module_file(filename, moddirlist)
31                              for filename in ext.sources ]
32 @@ -248,10 +254,15 @@ class PyBuildExt(build_ext):
33                  remove_modules.append(line[0])
34              input.close()
35  
36 +        print "remove_modules: ", remove_modules
37 +
38          for ext in self.extensions[:]:
39              if ext.name in remove_modules:
40                  self.extensions.remove(ext)
41  
42 +        for ext in self.extensions[:]:
43 +            print "extensions: ", ext.name
44 +
45          # When you run "make CC=altcc" or something similar, you really want
46          # those environment variables passed into the setup.py phase.  Here's
47          # a small set of useful ones.
48 @@ -1618,13 +1629,13 @@ class PyBuildExt(build_ext):
49  
50  
51          # Platform-specific libraries
52 -        if host_platform == 'linux2':
53 +        if host_platform == 'linux2' or host_platform == 'linux2-arm':
54              # Linux-specific modules
55              exts.append( Extension('linuxaudiodev', ['linuxaudiodev.c']) )
56          else:
57              missing.append('linuxaudiodev')
58  
59 -        if (host_platform in ('linux2', 'freebsd4', 'freebsd5', 'freebsd6',
60 +        if (host_platform in ('linux2','linux2-arm' 'freebsd4', 'freebsd5', 'freebsd6',
61                          'freebsd7', 'freebsd8')
62              or host_platform.startswith("gnukfreebsd")):
63              exts.append( Extension('ossaudiodev', ['ossaudiodev.c']) )
64 @@ -1755,6 +1766,10 @@ class PyBuildExt(build_ext):
65  ##         ext = Extension('xx', ['xxmodule.c'])
66  ##         self.extensions.append(ext)
67  
68 +#        print "missing: ", missing
69 +#        for ext in self.extensions:
70 +#            print "extensions: ", ext.name
71 +
72          return missing
73  
74      def detect_tkinter_explicitly(self):
75 @@ -2229,6 +2244,8 @@ Topic :: Software Development
76  """
77  
78  def main():
79 +    print "sys.path: ", sys.path
80 +    print "cross_compiling: ", cross_compiling
81      # turn off warnings when deprecated modules are imported
82      import warnings
83      warnings.filterwarnings("ignore",category=DeprecationWarning)

下一场执行./mk2_make.sh可以看来:

 1 sys.path:  ['/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13', '/home/pengdonglin/src/qemu/python_cross_compile/aarch32/build/lib.linux2-arm-2.7', '/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13/Lib', '/home/pengdonglin/src/qemu/python_cross_compile/Python-2.7.13/Lib/plat-linux2', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-x86_64-linux-gnu', '/usr/lib/python2.7/lib-tk', '/usr/lib/python2.7/lib-old', '/usr/lib/python2.7/lib-dynload']
 2 cross_compiling:  True
 3 build_extensions enter.
 4 extensions:  _struct
 5 extensions:  _ctypes_test
 6 extensions:  array
 7 extensions:  cmath
 8 extensions:  math
 9 extensions:  strop
10 extensions:  time
11 extensions:  datetime
12 extensions:  itertools
13 extensions:  future_builtins
14 extensions:  _random
15 extensions:  _collections
16 extensions:  _bisect
17 extensions:  _heapq
18 extensions:  operator
19 extensions:  _io
20 extensions:  _functools
21 extensions:  _json
22 extensions:  _testcapi
23 extensions:  _hotshot
24 extensions:  _lsprof
25 extensions:  unicodedata
26 extensions:  _locale
27 extensions:  fcntl
28 extensions:  pwd
29 extensions:  grp
30 extensions:  spwd
31 extensions:  select
32 extensions:  parser
33 extensions:  cStringIO
34 extensions:  cPickle
35 extensions:  mmap
36 extensions:  syslog
37 extensions:  audioop
38 extensions:  crypt
39 extensions:  _csv
40 extensions:  _socket
41 extensions:  _sha
42 extensions:  _md5
43 extensions:  _sha256
44 extensions:  _sha512
45 extensions:  termios
46 extensions:  resource
47 extensions:  nis
48 extensions:  binascii
49 extensions:  pyexpat
50 extensions:  _elementtree
51 extensions:  _multibytecodec
52 extensions:  _codecs_kr
53 extensions:  _codecs_jp
54 extensions:  _codecs_cn
55 extensions:  _codecs_tw
56 extensions:  _codecs_hk
57 extensions:  _codecs_iso2022
58 extensions:  _multiprocessing
59 extensions:  linuxaudiodev
60 extensions:  _ctypes
61 builtin_module_names:  ('__builtin__', '__main__', '_ast', '_bisect', '_codecs', '_collections', '_functools', '_heapq', '_io', '_locale', '_md5', '_random', '_sha', '_sha256', '_sha512', '_socket', '_sre', '_struct', '_symtable', '_warnings', '_weakref', 'array', 'binascii', 'cPickle', 'cStringIO', 'cmath', 'errno', 'exceptions', 'fcntl', 'gc', 'grp', 'imp', 'itertools', 'marshal', 'math', 'operator', 'posix', 'pwd', 'select', 'signal', 'spwd', 'strop', 'sys', 'syslog', 'thread', 'time', 'unicodedata', 'xxsubtype', 'zipimport', 'zlib')
62 remove_modules:  ['DESTLIB=$(LIBDEST)', 'MACHDESTLIB=$(BINLIBDEST)', 'DESTPATH=', 'SITEPATH=', 'TESTPATH=', 'MACHDEPPATH=:$(PLATDIR)', 'EXTRAMACHDEPPATH=', 'TKPATH=:lib-tk', 'OLDPATH=:lib-old', 'COREPYTHONPATH=$(DESTPATH)$(SITEPATH)$(TESTPATH)$(MACHDEPPATH)$(EXTRAMACHDEPPATH)$(TKPATH)$(OLDPATH)', 'PYTHONPATH=$(COREPYTHONPATH)', 'posix', 'errno', 'pwd', '_sre', '_codecs', '_weakref', 'zipimport', '_symtable', 'GLHACK=-Dclear=__GLclear', 'xxsubtype']
63 extensions:  _ctypes_test
64 extensions:  datetime
65 extensions:  future_builtins
66 extensions:  _json
67 extensions:  _testcapi
68 extensions:  _hotshot
69 extensions:  _lsprof
70 extensions:  parser
71 extensions:  mmap
72 extensions:  audioop
73 extensions:  crypt
74 extensions:  _csv
75 extensions:  termios
76 extensions:  resource
77 extensions:  nis
78 extensions:  pyexpat
79 extensions:  _elementtree
80 extensions:  _multibytecodec
81 extensions:  _codecs_kr
82 extensions:  _codecs_jp
83 extensions:  _codecs_cn
84 extensions:  _codecs_tw
85 extensions:  _codecs_hk
86 extensions:  _codecs_iso2022
87 extensions:  _multiprocessing
88 extensions:  linuxaudiodev
89 extensions:  _ctypes
90 Python build finished, but the necessary bits to build these modules were not found:
91 _bsddb             _curses            _curses_panel   
92 _sqlite3           _ssl               _tkinter        
93 bsddb185           bz2                dbm             
94 dl                 gdbm               imageop         
95 ossaudiodev        readline           sunaudiodev     
96 zlib                                                  
97 To find the necessary bits, look in setup.py in detect_modules() for the module's name.

以刚刚开头之时段,self.extensions中尚是都的,但是经过下面的处理后,
很多仓房都为remove了:

 1         for ext in self.extensions[:]:
 2             ext.sources = [ find_module_file(filename, moddirlist)
 3                             for filename in ext.sources ]
 4             if ext.depends is not None:
 5                 ext.depends = [find_module_file(filename, moddirlist)
 6                                for filename in ext.depends]
 7             else:
 8                 ext.depends = []
 9             # re-compile extensions if a header file has been changed
10             ext.depends.extend(headers)
11             # platform specific include directories
12             ext.include_dirs.extend(incdirlist)
13             # If a module has already been built statically,
14             # don't build it here
15             if ext.name in sys.builtin_module_names:
16                 self.extensions.remove(ext)

第15执行之笺注可以观看,如果sys.builtin_module_names中含有extensions中的库房,那么这库房就会从extensions中remove。从当下底解析看,交叉编译的下,setup.py的import
sys导入的相应是PC机上面的环境,导致sys.builtin_module_names的价为是PC上面python运行环境之值(可以于PC的终极下输入python,查看sys.builtin_module_names的值)。

2、 解决

这边为了简单起见,我们只是需要将方来问题之判断注释掉,如下:

 1 @@ -233,8 +239,8 @@ class PyBuildExt(build_ext):
 2  
 3              # If a module has already been built statically,
 4              # don't build it here
 5 -            if ext.name in sys.builtin_module_names:
 6 -                self.extensions.remove(ext)
 7 +            #if ext.name in sys.builtin_module_names:
 8 +            #    self.extensions.remove(ext)
 9  
10          # Parse Modules/Setup and Modules/Setup.local to figure out which
11          # modules are turned on in the file.

然后重新配置、编译、安装,
最后再制作ramdisk文件,启动板子sqlite,重新履行下的测试:

 1 [root@vexpress ]# python /usr/lib/python2.7/test/test___all__.py
 2 test_all (__main__.AllTest) ... BaseHTTPServer
 3 Bastion
 4 CGIHTTPServer
 5 ConfigParser
 6 Cookie
 7 DocXMLRPCServer
 8 HTMLParser
 9 MimeWriter
10 Queue
11 SimpleHTTPServer
12 ... ...
13 'xml.sax.xmlreader', 'xmllib', 'xmlrpclib']
14 Following modules failed to be imported: ['ctypes.wintypes', 'dbhash', 'gzip', 'idlelib.AutoComplete']
15 ok
16 ----------------------------------------------------------------------
17 Ran 1 test in 9.345s
18 OK

好望,测试成功了。

下面开始同sqlite3交板子上面,同时深受python也增多多sqlite3底支持。

3、支持sqlite3

首先到http://www.sqlite.org/download.html 下充斥最新的sqlite3的源码,这里自己用底是sqlite-autoconf-3170000.tar.gz,然后开展接力编译,下面是穿插编译的脚本mk.sh:

1 #!/bin/bash
2 export PATH=/home/pengdonglin/src/qemu/aarch32/arm-2014.05/bin:$PATH
3 
4 ../sqlite-autoconf-3170000/configure --host=arm-none-linux-gnueabi \
5     --prefix=`pwd`
6 
7 make -j4
8 make install

然后修改制作ramdisk的剧本:

 1 #!/bin/bash
 2 
 3 sudo rm -rf rootfs
 4 sudo rm -rf tmpfs
 5 sudo rm -rf ramdisk*
 6 
 7 sudo mkdir rootfs
 8 sudo cp ../busybox-1.24.2/_install/*  rootfs/ -raf
 9 
10 sudo mkdir -p rootfs/proc/
11 sudo mkdir -p rootfs/sys/
12 sudo mkdir -p rootfs/tmp/
13 sudo mkdir -p rootfs/root/
14 sudo mkdir -p rootfs/var/
15 sudo mkdir -p rootfs/mnt/
16 
17 sudo cp etc rootfs/ -arf
18 
19 sudo cp -arf ../arm-2014.05/arm-none-linux-gnueabi/libc/lib rootfs/
20 
21 #python
22 sudo mkdir -p rootfs/usr
23 pushd rootfs/usr
24 sudo cp  -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/bin .
25 sudo cp  -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/lib .
26 sudo cp  -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/include .
27 sudo cp  -raf /home/pengdonglin/qemu/thiry_part/Python/aarch32/share .
28 sudo /home/pengdonglin/qemu/aarch32/arm-2014.05/bin/arm-none-linux-gnueabi-strip lib/python*
29 popd
30 
31 #sqlite3
32 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/bin/* rootfs/bin/
33 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/include/* rootfs/include/
34 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/lib/* rootfs/lib/
35 sudo cp -raf /home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/share/* rootfs/usr/share
36 
37 
38 sudo mkdir -p rootfs/dev/
39 sudo mknod rootfs/dev/tty1 c 4 1
40 sudo mknod rootfs/dev/tty2 c 4 2
41 sudo mknod rootfs/dev/tty3 c 4 3
42 sudo mknod rootfs/dev/tty4 c 4 4
43 sudo mknod rootfs/dev/console c 5 1
44 sudo mknod rootfs/dev/null c 1 3
45 
46 sudo rm -rf rootfs/lib/*.a
47 sudo rm -rf rootfs/lib/*.la
48 sudo ../arm-2014.05/bin/arm-none-linux-gnueabi-strip rootfs/lib/*
49 
50 sudo dd if=/dev/zero of=ramdisk bs=1M count=100
51 sudo mkfs.ext4 -F ramdisk
52 
53 sudo mkdir -p tmpfs
54 sudo mount -t ext4 ramdisk ./tmpfs/  -o loop
55 sudo cp -raf rootfs/*  tmpfs/
56 sudo umount tmpfs
57 
58 sudo gzip --best -c ramdisk > ramdisk.gz
59 sudo mkimage -n "ramdisk" -A arm -O linux -T ramdisk -C gzip -d ramdisk.gz ramdisk.img

这么于板子上面就是可利用sqlite3了,但是此时python还无法用,需要再次编译python,并点名sqlite3的lib和include的门径,修改mk1_config.sh如下:

#!/bin/bash

export PATH=/home/pengdonglin/qemu/aarch32/arm-2014.05/bin:$PATH

../Python-2.7.13/configure --prefix=`pwd` \
    --host=arm-none-linux-gnueabi \
    --build=x86_64-linux-gnu \
    --enable-ipv6 \
    --enable-shared \
    ac_cv_file__dev_ptmx="yes" \
    ac_cv_file__dev_ptc="no" \
    LDFLAGS="-L/home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/lib" \
    CPPFLAGS="-I/home/pengdonglin/qemu/thiry_part/SQlite3/aarch32/include"

这般在Makefile调用setup.py时就是会见将sqlite3相关的模块编译进来,然后再度执行mk2_make.sh和mk3_install.sh,然后我们得检查一下:

1 $ls aarch32/build/lib.linux2-arm-2.7/
2 array.so*       _codecs_hk.so*       cPickle.so*       datetime.so*         _heapq.so*         _locale.so*          _multiprocessing.so*  _random.so*   _socket.so*        _sysconfigdata.pyc
3 audioop.so*     _codecs_iso2022.so*  crypt.so*         _elementtree.so*     _hotshot.so*       _lsprof.so*          nis.so*               resource.so*  spwd.so*           syslog.so*
4 binascii.so*    _codecs_jp.so*       cStringIO.so*     fcntl.so*            _io.so*            math.so*             operator.so*          select.so*    _sqlite3.so*       termios.so*
5 _bisect.so*     _codecs_kr.so*       _csv.so*          _functools.so*       itertools.so*      _md5.so*             ossaudiodev.so*       _sha256.so*   strop.so*          _testcapi.so*
6 cmath.so*       _codecs_tw.so*       _ctypes.so*       future_builtins.so*  _json.so*          mmap.so*             parser.so*            _sha512.so*   _struct.so*        time.so*
7 _codecs_cn.so*  _collections.so*     _ctypes_test.so*  grp.so*              linuxaudiodev.so*  _multibytecodec.so*  pyexpat.so*           _sha.so*      _sysconfigdata.py  unicodedata.so*

好观看,库已经好咸了。

4、测试

再制作ramdisk文件,启动系统。

编排测试sqlite3的脚本sq_demo.py如下:

 1 #!/usr/bin/python
 2               
 3 import sqlite3                   
 4                                      
 5 #open database                     
 6 conn = sqlite3.connect('test.db')
 7 print "Opened database successfully";
 8                                      
 9 conn.execute('''CREATE TABLE COMPANY 
10        (ID INT PRIMARY KEY     NOT NULL,
11        NAME           TEXT    NOT NULL, 
12        AGE            INT     NOT NULL, 
13        ADDRESS        CHAR(50),         
14        SALARY         REAL);''')       
15 print "Table created successfully";    
16                                    
17 #insert                            
18 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
19       VALUES (1, 'Paul', 32, 'California', 20000.00 )");        
20                                                                 
21 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
22       VALUES (2, 'Allen', 25, 'Texas', 15000.00 )");            
23                                                                 
24 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
25       VALUES (3, 'Teddy', 23, 'Norway', 20000.00 )");           
26                                                                 
27 conn.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
28       VALUES (4, 'Mark', 25, 'Rich-Mond ', 65000.00 )");        
29                                                                 
30 conn.commit()                                                   
31 print "Records created successfully";                   
32                                      
33 #select                              
34 cursor = conn.execute("SELECT id, name, address, salary  from COMPANY")
35 for row in cursor:                                                     
36    print "ID = ", row[0]                                               
37    print "NAME = ", row[1]                                             
38    print "ADDRESS = ", row[2]
39    print "SALARY = ", row[3], "\n"
40                                   
41 print "Operation done successfully";
42                                     
43 #delect                             
44 conn.execute("DELETE from COMPANY where ID=2;")
45 conn.commit()                                  
46 print "Total number of rows deleted :", conn.total_changes
47                                                           
48 cursor = conn.execute("SELECT id, name, address, salary  from COMPANY")
49 for row in cursor:                                                     
50    print "ID = ", row[0]                                               
51    print "NAME = ", row[1]                                             
52    print "ADDRESS = ", row[2]
53    print "SALARY = ", row[3], "\n"
54                                   
55 print "Operation done successfully";
56                                     
57 conn.close()

脚是出口结果:

[root@vexpress ]# python /tmp/sq_demo.py 
Opened database successfully
Table created successfully
Records created successfully
ID =  1
NAME =  Paul
ADDRESS =  California
SALARY =  20000.0 

ID =  2
NAME =  Allen
ADDRESS =  Texas
SALARY =  15000.0 

ID =  3
NAME =  Teddy
ADDRESS =  Norway
SALARY =  20000.0 

ID =  4
NAME =  Mark
ADDRESS =  Rich-Mond 
SALARY =  65000.0 

Operation done successfully
Total number of rows deleted : 5
ID =  1
NAME =  Paul
ADDRESS =  California
SALARY =  20000.0 

ID =  3
NAME =  Teddy
ADDRESS =  Norway
SALARY =  20000.0 

ID =  4
NAME =  Mark
ADDRESS =  Rich-Mond 
SALARY =  65000.0 

Operation done successfully

完。

网站地图xml地图