使⽤爬⾍爬取某⼀电商平台上部分商品信息,包括但不限于商品图⽚、价格、名称等。
顾客
商家
管理员
后端:Python v3.8 、Golang v1.15
数据库:MySql v5.7.30、Redis v4.0.9
文件存储 :七牛云存储
支付接口:支付FM
class product_img_info(Base):
__tablename__ = 'product_param_img' # 数据库中的表名
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String(), nullable=False)
title = Column(String())
category_id = Column(String())
product_id = Column(String())
info = Column(String())
img_path = Column(String())
price = Column(String())
discount_price = Column(String())
created_at = Column(DateTime, default=datetime.now)
updated_at = Column(DateTime, default=datetime.now)
deleted_at = Column(DateTime, default = None)
def __repr__(self):
return """
<product_img_info(id:%s, product_id:%s>
""" % (self.id,self.product_id)
def getHTMLText(url):
try:
header = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36',
'cookie': '' # 到浏览器复制cookie
}
r = requests.get(url, timeout=30, headers=header)
r.raise_for_status()
r.encoding = r.apparent_encoding
# print(r.text)
return r.text
except:
return ""
def parsePage(html):
view_prices = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
view_fees = re.findall(r'\"view_fee\"\:\"[\d\.]*\"', html)
raw_titles = re.findall(r'\"raw_title\"\:\".*?\"', html)
titles = re.findall(r'\"title\"\:\".*?\"', html)
user_ids = re.findall(r'\"user_id\"\:\"[\d\.]*\"',html)
pic_urls = re.findall(r'\"pic_url\"\:\".*?\"',html)
detail_urls = re.findall(r'\"detail_url\"\:\".*?\"',html)
for view_price,view_fee,title,raw_title,user_id,pic_url,detail_url in zip(view_prices,view_fees,titles,raw_titles,user_ids,pic_urls,detail_urls):
price=eval(view_price.split(':')[1])
discount_price=eval(view_fee.split(':')[1])
name=eval(title.split(':')[1])
a4=eval(raw_title.split(':')[1])
product_id=eval(user_id.split(':')[1])
img_path=eval(pic_url.split(':')[1])
persopn = product_img_info(name=name,title=name,product_id=product_id,category_id="6",info=name,img_path=img_path,price=price,discount_price=discount_price)
session.add(persopn) # 增加一个
session.commit() # 提交到数据库中
# 1手机 2女装 3电脑 4杯子 5零食 6耳机
def main():
goods = '耳机'
depth = 1
start_url = 'https://s.taobao.com/search?q=' + goods
for i in range(depth):
try:
url = start_url + '&s=' + str(44 * i)
html = getHTMLText(url)
parsePage(html)
except:
continue
if __name__ == '__main__':
# Base.metadata.create_all() # 创建表需要执行这行代码,如果表存在则不创建
#sqlOperation()
main()
部分数据库建设
//User 用户模型
type User struct {
gorm.Model
UserName string `gorm:"unique"`
Email string //`gorm:"unique"`
PasswordDigest string
Nickname string `gorm:"unique"`
Status string
Limit int // 0 非管理员 1 管理员
Type int // 0表示用户 1表示商家
Avatar string `gorm:"size:1000"`
Monery int
}
//商品模型
type Product struct {
gorm.Model
ProductID string `gorm:"primary_key"`
Name string
CategoryID int
Title string
Info string `gorm:"size:1000"`
ImgPath string
Price string
DiscountPrice string
OnSale string
Num int
BossID int
BossName string
BossAvatar string
}
部分逻辑代码
func (service *UpProductService) UpProduct() serializer.Response {
var product model.Product
code := e.SUCCESS
err := model.DB.First(&product,service.ID).Error
if err != nil {
logging.Info(err)
code = e.ErrorDatabase
return serializer.Response{
Status: code,
Msg: e.GetMsg(code),
Error: err.Error(),
}
}
product.OnSale = service.OnSale
err = model.DB.Save(&product).Error
if err != nil {
logging.Info(err)
code = e.ErrorDatabase
return serializer.Response{
Status: code,
Msg: e.GetMsg(code),
Error: err.Error(),
}
}
return serializer.Response{
Status: code,
Data: serializer.BuildProduct(product),
Msg: e.GetMsg(code),
}
}
//更新商品
func (service *UpdateProductService) Update() serializer.Response {
product := model.Product{
Name: service.Name,
CategoryID: service.CategoryID,
Title: service.Title,
Info: service.Info,
ImgPath: service.ImgPath,
Price: service.Price,
DiscountPrice: service.DiscountPrice,
OnSale: service.OnSale,
}
product.ID = service.ID
code := e.SUCCESS
err := model.DB.Save(&product).Error
if err != nil {
logging.Info(err)
code = e.ErrorDatabase
return serializer.Response{
Status: code,
Msg: e.GetMsg(code),
Error: err.Error(),
}
}
return serializer.Response{
Status: code,
Msg: e.GetMsg(code),
}
}
import axios from 'axios'
// 创建商品
const postProduct = form =>
axios.post('/api/v1/products', form).then(res => res.data)
// 读商品详情
const showProduct = id =>
axios.get(`/api/v1/products/${id}`).then(res => res.data)
// 读取商品列表
const listProducts = (category_id, start, limit) =>
axios
.get('/api/v1/products', { params: { category_id, start, limit } })
.then(res => res.data)
//读取商品的图片
const showPictures = id => axios.get(`/api/v1/imgs/${id}`).then(res => res.data)
//搜索商品
const searchProducts = form =>
axios.post('/api/v1/searches', form).then(res => res.data)
export {
postProduct,
showProduct,
listProducts,
showPictures,
searchProducts
}
发布商品
购物车
结算页面
个人中心
这个商场是在作者的开源项目基础上加以改进的!
原GitHub地址:CongZ666
真的非常感谢作者的开源
让我能通过这个项目,懂得Gin+Vue
前后端分离的很多知识!!
还有懂了一些如同支付
功能等,之前没有做过的功能。
不过我还没搞懂极验
的功能。后续再完善。
小生凡一,期待你的关注。
联系客服