利用Python脚本生成sitemap.xml的实现方法

yun

最近项目中需要用脚本生成sitemap，中间学习了一下sitemap的格式和lxml库的用法。把结果记录一下，方便以后需要直接拿来用。下面这篇文章主要介绍了利用Python脚本生成sitemap.xml的实现方法,需要的朋友可以参考借鉴，一起来看看吧。

安装lxml

首先需要pip install lxml安装lxml库。

如果你在ubuntu上遇到了以下错误:

1. #include "libxml/xmlversion.h"
   
2. compilation terminated.
   
3. error: command 'x86_64-linux-gnu-gcc' failed with exit status 1
   
4. ----------------------------------------
   
5. Cleaning up...
   
6. Removing temporary dir /tmp/pip_build_root...
   
7. Command /usr/bin/python -c "import setuptools, tokenize;__file__='/tmp/pip_build_root/lxml/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record /tmp/pip-O4cIn6-record/install-record.txt --single-version-externally-managed --compile failed with error code 1 in /tmp/pip_build_root/lxml
   
8. Exception information:
   
9. Traceback (most recent call last):
   
10. File "/usr/lib/python2.7/dist-packages/pip/basecommand.py", line 122, in main
    
11.   status = self.run(options, args)
    
12. File "/usr/lib/python2.7/dist-packages/pip/commands/install.py", line 283, in run
    
13.   requirement_set.install(install_options, global_options, root=options.root_path)
    
14. File "/usr/lib/python2.7/dist-packages/pip/req.py", line 1435, in install
    
15.   requirement.install(install_options, global_options, *args, **kwargs)
    
16. File "/usr/lib/python2.7/dist-packages/pip/req.py", line 706, in install
    
17.   cwd=self.source_dir, filter_stdout=self._filter_install, show_stdout=False)
    
18. File "/usr/lib/python2.7/dist-packages/pip/util.py", line 697, in call_subprocess
    
19.   % (command_desc, proc.returncode, cwd))
    
20. InstallationError: Command /usr/bin/python -c "import setuptools, tokenize;__file__='/tmp/pip_build_root/lxml/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record /tmp/pip-O4cIn6-record/install-record.txt --single-version-externally-managed --compile failed with error code 1 in /tmp/pip_build_root/lxml
    

复制代码

请安装以下依赖：

1. sudo apt-get install libxml2-dev libxslt1-dev
   

复制代码

Python代码

下面是生成sitemap和sitemapindex索引的代码，可以按照需求传入需要的参数，或者增加字段：

1. #!/usr/bin/env python
   
2. # -*- coding:utf-8 -*-
   
3. 
   
4. import io
   
5. import re
   
6. from lxml import etree
   
7. 
   
8. 
   
9. def generate_xml(filename, url_list):
   
10.   """Generate a new xml file use url_list"""
    
11.   root = etree.Element('urlset',
    
12.              xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
    
13.   for each in url_list:
    
14.     url = etree.Element('url')
    
15.     loc = etree.Element('loc')
    
16.     loc.text = each
    
17.     url.append(loc)
    
18.     root.append(url)
    
19. 
    
20.   header = u'<?xml version="1.0" encoding="UTF-8"?>\n'
    
21.   s = etree.tostring(root, encoding='utf-8', pretty_print=True)
    
22.   with io.open(filename, 'w', encoding='utf-8') as f:
    
23.     f.write(unicode(header+s))
    
24. 
    
25. 
    
26. def update_xml(filename, url_list):
    
27.   """Add new url_list to origin xml file."""
    
28.   f = open(filename, 'r')
    
29.   lines = [i.strip() for i in f.readlines()]
    
30.   f.close()
    
31. 
    
32.   old_url_list = []
    
33.   for each_line in lines:
    
34.     d = re.findall('<loc>(http:\/\/.+)<\/loc>', each_line)
    
35.     old_url_list += d
    
36.   url_list += old_url_list
    
37. 
    
38.   generate_xml(filename, url_list)
    
39. 
    
40. 
    
41. def generatr_xml_index(filename, sitemap_list, lastmod_list):
    
42.   """Generate sitemap index xml file."""
    
43.   root = etree.Element('sitemapindex',
    
44.              xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
    
45.   for each_sitemap, each_lastmod in zip(sitemap_list, lastmod_list):
    
46.     sitemap = etree.Element('sitemap')
    
47.     loc = etree.Element('loc')
    
48.     loc.text = each_sitemap
    
49.     lastmod = etree.Element('lastmod')
    
50.     lastmod.text = each_lastmod
    
51.     sitemap.append(loc)
    
52.     sitemap.append(lastmod)
    
53.     root.append(sitemap)
    
54. 
    
55.   header = u'<?xml version="1.0" encoding="UTF-8"?>\n'
    
56.   s = etree.tostring(root, encoding='utf-8', pretty_print=True)
    
57.   with io.open(filename, 'w', encoding='utf-8') as f:
    
58.     f.write(unicode(header+s))
    
59. 
    
60. 
    
61. if __name__ == '__main__':
    
62.   urls = ['http://www.baidu.com'] * 10
    
63.   mods = ['2004-10-01T18:23:17+00:00'] * 10
    
64.   generatr_xml_index('index.xml', urls, mods)

复制代码

效果

生成的效果应该是这种格式：

sitemap格式:

1. <?xml version="1.0" encoding="UTF-8"?>
   
2. <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   
3. <url>
   
4.   <loc>http://www.example.com/foo.html</loc>
   
5. </url>
   
6. </urlset>

复制代码

sitemapindex格式：

1. <?xml version="1.0" encoding="UTF-8"?>
   
2.   <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   
3.   <sitemap>
   
4.    <loc>http://www.example.com/sitemap1.xml.gz</loc>
   
5.    <lastmod>2017-02-01T18:23:17+00:00</lastmod>
   
6.   </sitemap>
   
7.   <sitemap>
   
8.    <loc>http://www.example.com/sitemap2.xml.gz</loc>
   
9.    <lastmod>2017-02-01</lastmod>
   
10.   </sitemap>
    
11.   </sitemapindex>

复制代码

lastmod时间格式的问题

格式是用ISO 8601的标准，如果是linux/unix系统，可以使用以下函数获取

1. def get_lastmod_time(filename):
   
2.   time_stamp = os.path.getmtime(filename)
   
3.   t = time.localtime(time_stamp)
   
4.   # return time.strftime('%Y-%m-%dT%H:%M:%S+08:00', t)
   
5.   return time.strftime('%Y-%m-%dT%H:%M:%SZ', t)
   

复制代码

优化

一般来说，用lxml效率低并且内存占用比较大，可以直接用文件的write方法创建。

1. def generate_xml(filename, url_list):
   
2.   with gzip.open(filename,"w") as f:
   
3.     f.write("""<?xml version="1.0" encoding="utf-8"?>
   
4. <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n""")
   
5.     for i in url_list:
   
6.       f.write("""<url><loc>%s</loc></url>\n"""%i)
   
7.     f.write("""</urlset>""")
   
8. 
   
9. 
   
10. def append_xml(filename, url_list):
    
11.   with gzip.open(filename, 'r') as f:
    
12.     for each_line in f:
    
13.       d = re.findall('<loc>(http:\/\/.+)<\/loc>', each_line)
    
14.       url_list.extend(d)
    
15. 
    
16.     generate_xml(filename, set(url_list))
    
17. 
    
18. 
    
19. def modify_time(filename):
    
20.   time_stamp = os.path.getmtime(filename)
    
21.   t = time.localtime(time_stamp)
    
22.   return time.strftime('%Y-%m-%dT%H:%M:%S:%SZ', t)
    
23. 
    
24. 
    
25. def new_xml(filename, url_list):
    
26.   generate_xml(filename, url_list)
    
27.   root = dirname(filename)
    
28. 
    
29.   with open(join(dirname(root), "sitemap.xml"),"w") as f:
    
30.     f.write('<?xml version="1.0" encoding="utf-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n')
    
31.     for i in glob.glob(join(root,"*.xml.gz")):
    
32.       lastmod = modify_time(i)
    
33.       i = i[len(CONFIG.SITEMAP_PATH):]
    
34.       f.write("<sitemap>\n<loc>http:/%s</loc>\n"%i)
    
35.       f.write("<lastmod>%s</lastmod>\n</sitemap>\n"%lastmod)
    
36.     f.write('</sitemapindex>')
    

复制代码

总结

以上就是这篇文章的全部内容了，希望本文的内容对大家学习或者使用python能带来一定的帮助

heimao

python精髓哈