image.png

    1. package config
    2. import (
    3. "context"
    4. "fmt"
    5. "github.com/aliyun/aliyun-oss-go-sdk/oss"
    6. "github.com/go-redis/redis/v8" //"github.com/gomodule/redigo/redis"
    7. _ "github.com/gofiber/fiber/v2"
    8. "github.com/olivere/elastic/v7"
    9. "github.com/programmerug/fibergorm/entities"
    10. "go.mongodb.org/mongo-driver/mongo"
    11. "go.mongodb.org/mongo-driver/mongo/options"
    12. "gorm.io/driver/mysql"
    13. "gorm.io/gorm"
    14. _ "gorm.io/hints"
    15. )
    16. var DB *gorm.DB
    17. var ES *elastic.Client
    18. var esUrl = "http://192.168.5.88:9205"
    19. var RDB *redis.Client
    20. var MONGO *mongo.Client
    21. var OssCli *oss.Client
    22. func Connect() (err error) {
    23. //初始化数据库连接
    24. dns := "root:123456@(localhost:3306)/test?charset=utf8mb4&parseTime=True&loc=Local"
    25. DB, err = gorm.Open(mysql.Open(dns), &gorm.Config{})
    26. if err != nil {
    27. fmt.Printf("连接mysql出错,错误信息:%v", err)
    28. } else {
    29. fmt.Println("成功连接mysql!")
    30. }
    31. return
    32. }
    33. func ConnectEs() (err error) {
    34. //连接客户端
    35. ES, err := elastic.NewClient(elastic.SetURL(esUrl), elastic.SetSniff(false))
    36. if err != nil {
    37. // Handle error
    38. panic(err)
    39. }
    40. // 获取版本号的直接API
    41. esVersion, err := ES.ElasticsearchVersion(esUrl)
    42. if err != nil {
    43. panic(err)
    44. fmt.Printf("连接es出错,错误信息:%v", err)
    45. } else {
    46. fmt.Println("成功连接es!")
    47. fmt.Printf("es的版本为%s\n", esVersion)
    48. }
    49. return
    50. }
    51. func ConnectRedis() (err error) {
    52. //Background返回一个非空的Context。 它永远不会被取消,没有值,也没有期限。
    53. //它通常在main函数,初始化和测试时使用,并用作传入请求的顶级上下文。
    54. var ctx = context.Background()
    55. RDB := redis.NewClient(&redis.Options{
    56. Addr: "192.168.5.86:8001",
    57. Password: "12345678",
    58. DB: 0,
    59. })
    60. _, err = RDB.Ping(ctx).Result()
    61. if err != nil {
    62. fmt.Printf("连接redis出错,错误信息:%v", err)
    63. } else {
    64. fmt.Println("成功连接redis!")
    65. }
    66. return
    67. }
    68. func ConnectMongoDB() (err error) {
    69. // Set client options 建立mongodb连接
    70. clientOptions := options.Client().ApplyURI("mongodb://127.0.0.1:27017/test")
    71. // Connect to MongoDB
    72. MONGO, err := mongo.Connect(context.TODO(), clientOptions)
    73. if err != nil {
    74. fmt.Printf("连接MongoDB出错,错误信息:%v", err)
    75. }
    76. err = MONGO.Ping(context.TODO(), nil)
    77. if err != nil {
    78. fmt.Printf("Ping MongoDB出错,错误信息:%v", err)
    79. } else {
    80. fmt.Println("成功连接MongoDB!")
    81. }
    82. return
    83. }
    84. //创建oss连接
    85. func ConnectOss() (err error) {
    86. var ossConfig entities.OssCon
    87. OssCli, err = oss.New(ossConfig.OssEndpoint, ossConfig.OssAccessKeyID, ossConfig.OssAccessKeySecret)
    88. if err != nil {
    89. panic(err)
    90. }else {
    91. fmt.Println("成功连接Oss!")
    92. }
    93. return
    94. }
    1. package entities
    2. import (
    3. "fmt"
    4. "log"
    5. _ "time"
    6. )
    7. type Todo struct {
    8. ID int `json:"id"`
    9. Tiltle string `json:"title"`
    10. Status bool `json:"status"`
    11. }
    12. type Language struct {
    13. Content string `json:"content"`
    14. }
    15. type OssCon struct {
    16. OssBucket string `json:"oss_bucket"`
    17. OssEndpoint string `json:"oss_endpoint"`
    18. // oss访问key
    19. OssAccessKeyID string `json:"oss_access_key_id"`
    20. // oss访问key secret
    21. OssAccessKeySecret string `json:"oss_access_key_secret"`
    22. }
    23. type Info struct {
    24. Name string
    25. Price int `json:"price" from:"pri"`
    26. }
    27. type DBDrive struct {
    28. Type string
    29. Mysql Mysql
    30. Badger Badger
    31. }
    32. type User struct {
    33. Id int `PK`
    34. Username string
    35. Pwd string
    36. }
    37. type Mysql struct{}
    38. type Badger struct{}
    39. func (m Mysql) GetGlobalObj() {
    40. fmt.Println("mysql func")
    41. }
    42. func (b Badger) GetGlobalObj() {
    43. fmt.Println("badger func")
    44. }
    45. //xxl.Logger接口实现
    46. type Logger struct{}
    47. func (l *Logger) Info(format string, a ...interface{}) {
    48. fmt.Println(fmt.Sprintf("自定义日志 - "+format, a...))
    49. }
    50. func (l *Logger) Error(format string, a ...interface{}) {
    51. log.Println(fmt.Sprintf("自定义日志 - "+format, a...))
    52. }
    53. type Student struct {
    54. Name string
    55. Age int
    56. }
    1. package handlers
    2. import (
    3. "bufio"
    4. "context"
    5. "encoding/json"
    6. "flag"
    7. "fmt"
    8. "github.com/aliyun/aliyun-oss-go-sdk/oss"
    9. "github.com/apache/rocketmq-client-go/v2"
    10. "github.com/apache/rocketmq-client-go/v2/consumer"
    11. "github.com/apache/rocketmq-client-go/v2/primitive"
    12. _ "github.com/apache/rocketmq-client-go/v2/primitive"
    13. "github.com/apache/rocketmq-client-go/v2/producer"
    14. _ "github.com/apache/rocketmq-client-go/v2/producer"
    15. "github.com/blugelabs/bluge"
    16. dlp "github.com/bytedance/godlp"
    17. "github.com/fatih/set"
    18. _ "github.com/go-redis/redis/v8"
    19. "github.com/gofiber/fiber/v2"
    20. "github.com/gofiber/fiber/v2/utils"
    21. "github.com/google/go-querystring/query"
    22. "github.com/kardianos/service"
    23. _ "github.com/olivere/elastic/v7"
    24. "github.com/pkg/errors"
    25. "github.com/programmerug/fibergorm/config"
    26. "github.com/programmerug/fibergorm/entities"
    27. "github.com/xxl-job/xxl-job-executor-go"
    28. "github.com/xxl-job/xxl-job-executor-go/example/task"
    29. "go.mongodb.org/mongo-driver/bson"
    30. "log"
    31. "os"
    32. "reflect"
    33. "runtime"
    34. "strconv"
    35. "testing"
    36. "time"
    37. )
    38. func GetDogs(c *fiber.Ctx) error {
    39. var dogs []entities.Todo
    40. config.DB.Find(&dogs)
    41. return c.Status(200).JSON(dogs)
    42. }
    43. func GetDog(c *fiber.Ctx) error {
    44. id := c.Params("id")
    45. results := utils.ImmutableString(c.Params("id"))
    46. log.Println("打印格式化参数:" + results)
    47. //拷贝相关参数
    48. // Make a copy
    49. buffer := make([]byte, len(results))
    50. copy(buffer, results)
    51. resultCopy := string(buffer)
    52. log.Println("打印拷贝参数:" + resultCopy)
    53. var dog entities.Todo
    54. result := config.DB.Find(&dog, id)
    55. if result.RowsAffected == 0 {
    56. return c.SendStatus(404)
    57. }
    58. return c.Status(200).JSON(&dog)
    59. }
    60. func Get(c *fiber.Ctx) error {
    61. return c.Send([]byte("Hello, World!"))
    62. }
    63. func AddDog(c *fiber.Ctx) error {
    64. dog := new(entities.Todo)
    65. if err := c.BodyParser(dog); err != nil {
    66. return c.Status(503).SendString(err.Error())
    67. }
    68. config.DB.Create(&dog)
    69. return c.Status(201).JSON(dog)
    70. }
    71. func UpdateDog(c *fiber.Ctx) error {
    72. dog := new(entities.Todo)
    73. id := c.Params("id")
    74. if err := c.BodyParser(dog); err != nil {
    75. return c.Status(503).SendString(err.Error())
    76. }
    77. config.DB.Where("id = ?", id).Updates(&dog)
    78. return c.Status(200).JSON(dog)
    79. }
    80. func RemoveDog(c *fiber.Ctx) error {
    81. id := c.Params("id")
    82. var dog entities.Todo
    83. result := config.DB.Delete(&dog, id)
    84. if result.RowsAffected == 0 {
    85. return c.SendStatus(404)
    86. }
    87. return c.SendStatus(200)
    88. }
    89. func Create(c *fiber.Ctx) error {
    90. e1 := entities.Language{Content: "golang"}
    91. _, err := config.ES.Index().
    92. Index("es").
    93. Id("1").
    94. BodyJson(e1).
    95. Do(context.Background())
    96. if err != nil {
    97. panic(err)
    98. }
    99. return c.SendStatus(200)
    100. }
    101. func GetEs(c *fiber.Ctx) error {
    102. query, err := config.ES.Get().Index("es").Id("1").Do(context.Background())
    103. if err != nil {
    104. panic(err)
    105. }
    106. var result entities.Language
    107. if query.Found {
    108. err := json.Unmarshal(query.Source, &result)
    109. if err != nil {
    110. fmt.Println(err)
    111. }
    112. fmt.Println(result.Content)
    113. }
    114. return c.Status(200).JSON(result)
    115. }
    116. func DeleteEs(c *fiber.Ctx) error {
    117. //Background返回一个非空的Context。 它永远不会被取消,没有值,也没有期限。
    118. //它通常在main函数,初始化和测试时使用,并用作传入请求的顶级上下文。
    119. var ctx = context.Background()
    120. response, err := config.ES.Delete().Index("es").Id("1").Do(ctx)
    121. if err != nil {
    122. panic(err)
    123. }
    124. //打印删除返回数据
    125. fmt.Println(response)
    126. return c.SendStatus(200)
    127. }
    128. func DeleteRedis(c *fiber.Ctx) error {
    129. //Background返回一个非空的Context。 它永远不会被取消,没有值,也没有期限。
    130. //它通常在main函数,初始化和测试时使用,并用作传入请求的顶级上下文。
    131. var ctx = context.Background()
    132. n, err := config.RDB.Del(ctx, "key1", "key2").Result()
    133. if err != nil {
    134. panic(err)
    135. }
    136. fmt.Printf("成功删除了 %v 个\n", n)
    137. return c.SendStatus(200)
    138. }
    139. func GetRedis(c *fiber.Ctx) error {
    140. //Background返回一个非空的Context。 它永远不会被取消,没有值,也没有期限。
    141. //它通常在main函数,初始化和测试时使用,并用作传入请求的顶级上下文。
    142. var ctx = context.Background()
    143. vType, err := config.RDB.Type(ctx, "key").Result()
    144. if err != nil {
    145. panic(err)
    146. }
    147. return c.Status(200).JSON(vType)
    148. }
    149. func SetRedis(c *fiber.Ctx) error {
    150. //Background返回一个非空的Context。 它永远不会被取消,没有值,也没有期限。
    151. //它通常在main函数,初始化和测试时使用,并用作传入请求的顶级上下文。
    152. var ctx = context.Background()
    153. err := config.RDB.SetEX(ctx, "key", "value", time.Hour*2).Err()
    154. if err != nil {
    155. panic(err)
    156. }
    157. return c.SendStatus(200)
    158. }
    159. func GetMongoDB(c *fiber.Ctx) error {
    160. // 2, 选择数据库ichunt 3, 选择表cron_log
    161. connect := config.MONGO.Database("ichunt").Collection("cron_log")
    162. var result struct {
    163. Value float64
    164. }
    165. filter := bson.D{{"name", "pi"}}
    166. err := connect.FindOne(context.TODO(), filter).Decode(result)
    167. if err != nil {
    168. log.Fatal(err)
    169. }
    170. fmt.Printf("Found a single document: %+v\n", result)
    171. return c.Status(200).JSON(result)
    172. }
    173. func Consumer(c *fiber.Ctx) error {
    174. // 设置推送消费者
    175. pull, _ := rocketmq.NewPushConsumer(
    176. //消费组
    177. consumer.WithGroupName("testGroup"),
    178. // namesrv地址
    179. consumer.WithNameServer([]string{"127.0.0.1:9876"}),
    180. )
    181. // 必须先在 开始前
    182. err := pull.Subscribe("Topic-test", consumer.MessageSelector{}, func(ctx context.Context, ext ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
    183. for i := range ext {
    184. fmt.Printf("subscribe callback:%v \n", ext[i])
    185. }
    186. return consumer.ConsumeSuccess, nil
    187. })
    188. if err != nil {
    189. fmt.Println(err.Error())
    190. }
    191. err = pull.Start()
    192. if err != nil {
    193. fmt.Println(err.Error())
    194. os.Exit(-1)
    195. }
    196. time.Sleep(time.Hour)
    197. err = pull.Shutdown()
    198. if err != nil {
    199. fmt.Printf("shutdown Consumer error:%s",err.Error())
    200. }
    201. return c.SendStatus(200)
    202. }
    203. func Producer(c *fiber.Ctx) error {
    204. push, _ := rocketmq.NewProducer(
    205. // 设置 nameSrvAddr
    206. // nameSrvAddr 是 Topic 路由注册中心
    207. producer.WithNameServer([]string{"127.0.0.1:9876"}),
    208. // 指定发送失败时的重试时间
    209. producer.WithRetry(2),
    210. // 设置 Group
    211. producer.WithGroupName("testGroup"),
    212. )
    213. // 开始连接
    214. err := push.Start()
    215. if err != nil {
    216. fmt.Printf("start producer error: %s", err.Error())
    217. os.Exit(1)
    218. }
    219. // 设置节点名称
    220. topic := "Topic-test"
    221. // 循坏发送信息 (同步发送)
    222. for i := 0; i < 10; i++ {
    223. msg := &primitive.Message{
    224. Topic: topic,
    225. Body: []byte("Hello RocketMQ Go Client" + strconv.Itoa(i)),
    226. }
    227. // 发送信息
    228. res, err := push.SendSync(context.Background(),msg)
    229. if err != nil {
    230. fmt.Printf("send message error:%s\n",err)
    231. }else {
    232. fmt.Printf("send message success: result=%s\n",res.String())
    233. }
    234. }
    235. // 关闭生产者
    236. err = push.Shutdown()
    237. if err != nil {
    238. fmt.Printf("shutdown producer error:%s",err.Error())
    239. }
    240. return c.SendStatus(200)
    241. }
    242. // Bucket: 获取bucket存储空间
    243. func Bucket() *oss.Bucket {
    244. var ossConfig entities.OssCon
    245. bucket, err := config.OssCli.Bucket(ossConfig.OssBucket)
    246. if err != nil {
    247. log.Println(err.Error())
    248. return nil
    249. }
    250. return bucket
    251. }
    252. func Upload(c *fiber.Ctx) error{
    253. filename := "hes.text"
    254. ossPath := "oss/" + filename // oss不能以/开头
    255. file, _ := os.Open(filename)
    256. err := Bucket().PutObject(ossPath, file)
    257. if err != nil {
    258. fmt.Printf("上传oss失败: %+v\n", err.Error())
    259. return c.SendStatus(503)
    260. }
    261. return c.SendStatus(200)
    262. }
    263. // 临时下载授权
    264. func DownUrl(c *fiber.Ctx) error{
    265. filepath := "oss/hes.text"
    266. url, err := Bucket().SignURL(filepath, oss.HTTPGet, 3600)
    267. if err != nil {
    268. fmt.Printf("下载oss失败: %+v\n", err.Error())
    269. return c.SendStatus(503)
    270. }
    271. return c.Status(200).JSON(url)
    272. }
    273. //运行相关参数
    274. func GetTest(c *fiber.Ctx) error{
    275. var name = flag.String("name", "everyon", "object")
    276. names := "robot"
    277. fmt.Printf("---:%s\n", names)
    278. fmt.Printf("-----:%s\n", *name)
    279. //var container = []string{"zero", "one", "two"}
    280. container := map[int]string{0: "zero", 1: "one", 2: "two"}
    281. fmt.Printf("The element is %q.\n", container[1])
    282. block := "function"
    283. {
    284. block := "inner"
    285. fmt.Printf("The block is %s.\n", block)
    286. }
    287. fmt.Printf("The block is %s.\n", block)
    288. s1 := make([]int, 5)
    289. fmt.Printf("The length of s1: %d\n", len(s1))
    290. fmt.Printf("The capacity of s1: %d\n", cap(s1))
    291. fmt.Printf("The value of s1: %d\n", s1)
    292. s2 := make([]int, 5, 8)
    293. fmt.Printf("The length of s2: %d\n", len(s2))
    294. fmt.Printf("The capacity of s2: %d\n", cap(s2))
    295. fmt.Printf("The value of s2: %d\n", s2)
    296. return c.SendStatus(200)
    297. }
    298. func StaticHtml(c *fiber.Ctx) error {
    299. if c.Path() == "/" || c.Path() == "" || c.Path() == "/index.html" {
    300. ht := `<!DOCTYPE html>
    301. <html lang="en">
    302. <head>
    303. <meta charset="UTF-8">
    304. <meta name="viewport" content="width=device-width, initial-scale=1.0">
    305. <meta http-equiv="X-UA-Compatible" content="ie=edge">
    306. <title>go api doc</title>
    307. <body>
    308. <h3>hhh</h3>
    309. <hr></hr>
    310. <h1>卧槽 哈哈哈哈哈</h1>
    311. </body>
    312. </html>`
    313. c.Response().Header.SetContentType("text/html; charset=utf-8") // 以HTML的方式解析显示
    314. return c.Send([]byte(ht))
    315. } else{
    316. return c.Send([]byte("404"))
    317. }
    318. }
    319. func SetHeader() map[string]string {
    320. headers := make(map[string]string)
    321. headers["X-Auth-AppId"] = "Auth.AppID"
    322. headers["X-Auth-Token"] = "common.GlobalObjs.User.Token"
    323. headers["X-Client-ID"] = "common.GlobalVar.ID"
    324. return headers
    325. }
    326. //将对象解析成sql
    327. func reflectSql(data interface{}) (error, string) {
    328. v := reflect.ValueOf(data)
    329. t := reflect.TypeOf(data)
    330. if t.Kind() != reflect.Struct {
    331. return errors.New("unsupported argument type"), ""
    332. }
    333. sql := "insert into " + t.Name()
    334. sqlEle := " ("
    335. sqlValue := " values("
    336. fieldNum := t.NumField()
    337. for i := 0; i < fieldNum; i++ {
    338. switch t.Field(i).Type.Kind() {
    339. case reflect.Int:
    340. if i == fieldNum-1 {
    341. sqlEle += t.Field(i).Name
    342. sqlValue += fmt.Sprintf("%d", v.Field(i).Int())
    343. } else {
    344. sqlEle += t.Field(i).Name + ","
    345. sqlValue += fmt.Sprintf("%d,", v.Field(i).Int())
    346. }
    347. case reflect.String:
    348. if i == fieldNum-1 {
    349. sqlEle += t.Field(i).Name
    350. sqlValue += v.Field(i).String()
    351. } else {
    352. sqlEle += t.Field(i).Name + ","
    353. sqlValue += v.Field(i).String() + ","
    354. }
    355. }
    356. }
    357. sqlEle += ")"
    358. sqlValue += ")"
    359. sql += sqlEle + sqlValue
    360. return nil, sql
    361. }
    362. func SaveSql(c *fiber.Ctx) error{
    363. info := entities.Info{
    364. Name: "name1",
    365. Price: 1,
    366. }
    367. err, sql := reflectSql(info)
    368. fmt.Println(err, sql) // <nil> insert into Info (Name,Price) values(name1,1)
    369. v, err := query.Values(info) // 会自动a-z排序、转义
    370. fmt.Println(v.Encode(), err) // age=2&comments=%E6%B5%8B%E8%AF%95&name=aq <nil>
    371. section1 := []string{"33", "22", "11"}
    372. section2 := []string{"33", "22","66"}
    373. hh := DifferenceString(section1, section2)
    374. fmt.Println(hh) // [11]
    375. return c.SendStatus(200)
    376. }
    377. // DifferenceString 取字符串切片的差集 (section1存在,不存在section2)
    378. func DifferenceString(section1, section2 []string) (intersection []string) {
    379. if len(section1) == 0 || len(section2) == 0 {
    380. return section1
    381. }
    382. a := set.New(set.ThreadSafe)
    383. b := set.New(set.ThreadSafe)
    384. for _, v := range section1 {
    385. a.Add(v)
    386. }
    387. for _, v := range section2 {
    388. b.Add(v)
    389. }
    390. intersectionSet := set.Difference(a, b)
    391. return set.StringSlice(intersectionSet)
    392. }
    393. // 取phone交集
    394. func intersectionPhone(section1, section2 []string) (intersection []string) {
    395. if len(section1) == 0 || len(section2) == 0 {
    396. return nil
    397. }
    398. a := set.New(set.ThreadSafe)
    399. b := set.New(set.ThreadSafe)
    400. for _, v := range section1 {
    401. a.Add(v)
    402. }
    403. for _, v := range section2 {
    404. b.Add(v)
    405. }
    406. intersectionSet := set.Intersection(a, b)
    407. return set.StringSlice(intersectionSet)
    408. }
    409. //超时控制
    410. func Close() {
    411. ch := make(chan struct{})
    412. go func() {
    413. // TODO s.Close()
    414. time.Sleep(time.Second*2)
    415. close(ch)
    416. }()
    417. select {
    418. case <-ch:
    419. fmt.Println("close ok")
    420. case <-time.After(time.Second):
    421. fmt.Println("close timeout")
    422. }
    423. }
    424. //获取传入的时间所在月份的第一天,即某月第一天的0点。如传入time.Now(), 返回当前月份的第一天0点时间。
    425. func GetFirstDateOfMonth(d time.Time) time.Time {
    426. d = d.AddDate(0, 0, -d.Day()+1)
    427. return GetZeroTime(d)
    428. }
    429. //获取传入的时间所在月份的最后一天,即某月最后一天的23:59:59。如传入time.Now(), 返回当前月份的最后一天0点时间。
    430. func GetLastDateOfMonth(d time.Time) time.Time {
    431. d = d.AddDate(0, 0, -d.Day()+1)
    432. d = Get23Time(d)
    433. return d.AddDate(0, 1, -1)
    434. }
    435. //获取某一天的0点时间
    436. func GetZeroTime(d time.Time) time.Time {
    437. return time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, d.Location())
    438. }
    439. //获取某一天的0点时间
    440. func Get23Time(d time.Time) time.Time {
    441. return time.Date(d.Year(), d.Month(), d.Day(), 23, 59, 59, 0, d.Location())
    442. }
    443. func Reflect() {
    444. hh := entities.DBDrive{
    445. Type: "Badger",
    446. }
    447. checkReflect(hh, hh.Type, "GetGlobalObj")
    448. }
    449. // inter一个嵌套结构体
    450. func checkReflect(inter interface{}, attributeName, method string) {
    451. v := reflect.ValueOf(inter)
    452. dbObj := v.FieldByName(attributeName) // 根据字段名获取属性 // 获得struct inter内嵌套的struct attributeName
    453. m := dbObj.MethodByName(method) // 获取 struct attributeName的method方法
    454. m.Call([]reflect.Value{}) // 调用struct attributeName的method方法 m.Call([]reflect.Value{reflect.ValueOf("参数值")})
    455. }
    456. type Program struct{}
    457. func (p *Program) Start(s service.Service) error {
    458. fmt.Println("server start")
    459. go p.run()
    460. return nil
    461. }
    462. func (p *Program) run() {
    463. fmt.Println("开机自启动服务 - run")
    464. }
    465. func (p *Program) Stop(s service.Service) error {
    466. fmt.Println("server stop")
    467. return nil
    468. }
    469. func runTime() {
    470. NCPU := runtime.NumCPU()
    471. runtime.GOMAXPROCS(NCPU)
    472. serConfig := &service.Config{
    473. Name: "hhhh",
    474. DisplayName: "test2",
    475. Description: "test 开机自启动",
    476. }
    477. pro := &Program{}
    478. s, err := service.New(pro, serConfig)
    479. if err != nil {
    480. fmt.Println(err, "service.New() err")
    481. }
    482. if len(os.Args) > 1 {
    483. if os.Args[1] == "install" {
    484. err = s.Install()
    485. if err != nil {
    486. fmt.Println("install err", err)
    487. } else {
    488. fmt.Println("install success")
    489. }
    490. return
    491. }
    492. if os.Args[1] == "remove" {
    493. err = s.Uninstall()
    494. if err != nil {
    495. fmt.Println("Uninstall err", err)
    496. } else {
    497. fmt.Println("Uninstall success")
    498. }
    499. return
    500. }
    501. }
    502. err = s.Run() // 运行服务
    503. if err != nil {
    504. fmt.Println("s.Run err", err)
    505. }
    506. }
    507. //寻找最长不含有重复字符的子串
    508. func lengthOf(s string) int {
    509. lastOccurred := make(map[rune]int)
    510. start := 0
    511. maxLength := 0
    512. for i, ch := range []rune(s) {
    513. if lastI, ok := lastOccurred[ch]; ok && lastI >= start {
    514. start = lastI + 1
    515. }
    516. if i - start + 1 > maxLength {
    517. maxLength = i - start + 1
    518. }
    519. lastOccurred[ch] = i
    520. }
    521. return maxLength
    522. }
    523. // GetWeekDay returns the week day name of a week day index.
    524. func GetWeekDay(index int) string {
    525. if index < 0 || index > 6 {
    526. return "Unknown"
    527. }
    528. weekDays := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
    529. return weekDays[index]
    530. }
    531. // table-driven 单测法,就是将流程沉淀为一个可复用的模板、并交由机器自动生成;
    532. // 人类则只需要准备数据部分,将自己的多条不同的数据一行行填充到表里,交给流程模板
    533. // 去构造子测试用例、查表、跑数据、比对结果,写单测这事就大功告成了。
    534. func TestGetWeekDay(t *testing.T) {
    535. // a subtest named "index=0"
    536. t.Run("index=0", func(t *testing.T) {
    537. index := 0
    538. want := "Sunday"
    539. if got := GetWeekDay(index); got != want {
    540. t.Errorf("GetWeekDay() = %v, want %v", got, want)
    541. }
    542. })
    543. // a subtest named "index=1"
    544. t.Run("index=1", func(t *testing.T) {
    545. index := 1
    546. want := "Monday"
    547. if got := GetWeekDay(index); got != want {
    548. t.Errorf("GetWeekDay() = %v, want %v", got, want)
    549. }
    550. })
    551. }
    552. func TestGetWeekDay1(t *testing.T) {
    553. type args struct {
    554. index int
    555. }
    556. tests := []struct {
    557. name string
    558. args args
    559. want string
    560. }{
    561. //todo: add test case
    562. }
    563. for _, tt := range tests {
    564. t.Run(tt.name, func(t *testing.T) {
    565. if got := GetWeekDay(tt.args.index); got != tt.want {
    566. t.Errorf("getweekday() = %v, want %v", got, tt.want)
    567. }
    568. })
    569. }
    570. }
    571. // 创建索引
    572. func writeIndex(indexPath string) {
    573. config := bluge.DefaultConfig(indexPath)
    574. writer, err := bluge.OpenWriter(config)
    575. if err != nil {
    576. log.Fatalf("error opening writer: %v", err)
    577. }
    578. defer writer.Close()
    579. // 新建文档
    580. doc := bluge.NewDocument("example").
    581. AddField(bluge.NewTextField("name", "bluge")).AddField(bluge.NewDateTimeField("created_at", time.Now()))
    582. err = writer.Update(doc.ID(), doc)
    583. if err != nil {
    584. log.Fatalf("error updating document: %v", err)
    585. }
    586. }
    587. // 批量创建
    588. func batch(indexPath string) {
    589. writer, err := bluge.OpenWriter(bluge.DefaultConfig(indexPath))
    590. batch := bluge.NewBatch()
    591. for i := 0; i < 10; i++ {
    592. doc := bluge.NewDocument(fmt.Sprintf("example_%d", i)).
    593. AddField(bluge.NewTextField(fmt.Sprintf("field_%d", i), fmt.Sprintf("value_%d", i%2))).AddField(bluge.NewDateTimeField("created_at", time.Now()))
    594. batch.Insert(doc)
    595. }
    596. err = writer.Batch(batch)
    597. if err != nil {
    598. log.Fatalf("error executing batch: %v", err)
    599. }
    600. batch.Reset()
    601. }
    602. // 查询
    603. func search(indexPath string) {
    604. config := bluge.DefaultConfig(indexPath)
    605. reader, err := bluge.OpenReader(config)
    606. if err != nil {
    607. log.Fatalf("error getting index reader: %v", err)
    608. }
    609. defer reader.Close()
    610. query := bluge.NewMatchQuery("value_1").SetField("field_1")
    611. request := bluge.NewTopNSearch(10, query).
    612. WithStandardAggregations()
    613. documentMatchIterator, err := reader.Search(context.Background(), request)
    614. if err != nil {
    615. log.Fatalf("error executing search: %v", err)
    616. }
    617. match, err := documentMatchIterator.Next()
    618. for err == nil && match != nil {
    619. err = match.VisitStoredFields(func(field string, value []byte) bool {
    620. fmt.Printf("match: %s:%s\n", field, string(value))
    621. return true
    622. })
    623. if err != nil {
    624. log.Fatalf("error loading stored fields: %v", err)
    625. }
    626. fmt.Println(match)
    627. match, err = documentMatchIterator.Next()
    628. }
    629. if err != nil {
    630. log.Fatalf("error iterator document matches: %v", err)
    631. }
    632. }
    633. func DlpDemoCtx(c *fiber.Ctx) error{
    634. dlpDemo()
    635. return c.SendStatus(200)
    636. }
    637. // 敏感信息参数过滤
    638. func dlpDemo() {
    639. caller := "replace.your.caller"
    640. if eng, err := dlp.NewEngine(caller); err == nil {
    641. eng.ApplyConfigDefault()
    642. fmt.Printf("DLP %s Demo:\n\n", eng.GetVersion())
    643. inStr := `我的邮件是abcd@abcd.com,
    644. 18612341234是我的电话
    645. 你家住在哪里啊? 我家住在北京市海淀区北三环西路43号,
    646. mac地址 06-06-06-aa-bb-cc
    647. 收件人:张真人 手机号码:13900000000`
    648. if outStr, _, err := eng.Deidentify(inStr); err == nil {
    649. fmt.Printf("\t1. Deidentify( inStr: %s )\n", inStr)
    650. fmt.Printf("\toutStr: %s\n", outStr)
    651. //eng.ShowResults(results)
    652. fmt.Println()
    653. }
    654. eng.Close()
    655. } else {
    656. fmt.Println("[dlp] NewEngine error: ", err.Error())
    657. }
    658. }
    659. func Reader(c *fiber.Ctx) error {
    660. reader := bufio.NewReader(os.Stdin)
    661. fmt.Println("请输入你想要输入的内容,按回车结束")
    662. fmt.Print("-> ")
    663. text, err := reader.ReadString('\n')
    664. if err != nil{
    665. panic(fmt.Errorf("发生致命错误: %w \n", err))
    666. }
    667. fmt.Println("刚接收到的内容是:",text)
    668. unix := time.Now().Unix();
    669. fmt.Println("time:", unix)
    670. return c.Send([]byte(text))
    671. }
    672. func XXLJob(c *fiber.Ctx) error{
    673. exec := xxl.NewExecutor(
    674. xxl.ServerAddr("http://dev-a-b2b-xxl-job.vandream.com/xxl-job-admin"),
    675. xxl.AccessToken(""), //请求令牌(默认为空)
    676. xxl.ExecutorIp(""), //可自动获取
    677. xxl.ExecutorPort("9156"), //默认9999(非必填)
    678. xxl.RegistryKey("obmp-apm"), //执行器名称
    679. xxl.SetLogger(&entities.Logger{}), //自定义日志
    680. )
    681. exec.Init()
    682. //设置日志查看handler
    683. exec.LogHandler(func(req *xxl.LogReq) *xxl.LogRes {
    684. return &xxl.LogRes{Code: 200, Msg: "", Content: xxl.LogResContent{
    685. FromLineNum: req.FromLineNum,
    686. ToLineNum: 2,
    687. LogContent: "SettlementCheckHandler",
    688. IsEnd: true,
    689. }}
    690. })
    691. //注册任务handler
    692. exec.RegTask("Task_Test", task.Test)
    693. defer exec.Run()
    694. return c.Next()
    695. }
    696. // 注册
    697. func SaveUser(c *fiber.Ctx) error {
    698. user := new(entities.User)
    699. if err := c.BodyParser(user); err != nil {
    700. return c.Status(503).SendString(err.Error())
    701. }
    702. config.DB.Create(&user)
    703. return c.Status(201).JSON(user)
    704. }
    705. //登录
    706. func ValidateUser(c *fiber.Ctx) error {
    707. username := c.Params("username")
    708. pwd := c.Params("pwd")
    709. var user entities.User
    710. result := config.DB.Find(&user, username, pwd)
    711. if result.RowsAffected == 0 {
    712. return errors.New("用户名或密码错误!")
    713. }
    714. return c.Status(200).JSON(user)
    715. }
    1. package main
    2. import (
    3. "encoding/json"
    4. "fmt"
    5. "github.com/gin-gonic/gin"
    6. "github.com/gofiber/fiber/v2"
    7. "github.com/gofiber/websocket/v2"
    8. "github.com/programmerug/fibergorm/config"
    9. _ "github.com/programmerug/fibergorm/config"
    10. "github.com/programmerug/fibergorm/entities"
    11. "github.com/programmerug/fibergorm/handlers"
    12. "github.com/pyroscope-io/pyroscope/pkg/agent/profiler"
    13. "io/ioutil"
    14. "log"
    15. "net/http"
    16. "time"
    17. )
    18. func main() {
    19. //fiberInit()
    20. ginInit()
    21. //监控 服务端: docker run -it -p 4040:4040 pyroscope/pyroscope:latest server
    22. // go get -u github.com/pyroscope-io/pyroscope/pkg/agent/profiler
    23. // Pyroscope 是一套开源的效能即时监控平台,简单的Server 及Agent 架构,让开发者可以轻松监控代码效能,不管你要找10 秒或几分钟内的效能数据,都可以快速的即时呈现,
    24. //开发者也不用在意装了此监控会造成任何效能上的负担。
    25. //Pyroscope 背后的储存采用 目前只有支援3 种语言(Python, Ruby 及Go) 未来会 假设您还没导入任何效能分析工具或平台,那Pyroscope 会是您最好的选择。
    26. profiler.Start(profiler.Config{
    27. ApplicationName: "simple.golang.app",
    28. ServerAddress: "http://localhost:4040",
    29. })
    30. }
    31. /**
    32. * fiber框架
    33. */
    34. func fiberInit(){
    35. app := fiber.New()
    36. config.Connect()
    37. app.Static("/", "/public")
    38. // => http://localhost:3000/js/script.js
    39. // => http://localhost:3000/css/style.css
    40. app.Static("/prefix", "/public")
    41. // => http://localhost:3000/prefix/js/script.js
    42. // => http://localhost:3000/prefix/css/style.css
    43. app.Static("*", "/public/index.html")
    44. // => http://localhost:3000/any/path/shows/index/html
    45. app.Get("/dogs", handlers.GetDogs)
    46. app.Get("/list", handlers.Get)
    47. app.Get("/dogs/:id", handlers.GetDog)
    48. app.Post("/dogs", handlers.AddDog)
    49. app.Put("/dogs/:id", handlers.UpdateDog)
    50. app.Delete("/dogs/:id", handlers.RemoveDog)
    51. app.Get("/test", handlers.GetTest)
    52. config.ConnectEs()
    53. //app.Get("/es", handlers.Create)
    54. app.Get("/es", handlers.GetEs)
    55. app.Delete("/deletes", handlers.DeleteEs)
    56. config.ConnectRedis()
    57. app.Put("/set", handlers.SetRedis)
    58. app.Get("/get", handlers.GetRedis)
    59. app.Delete("/deletes", handlers.DeleteRedis)
    60. config.ConnectMongoDB()
    61. app.Get("/mongodb", handlers.GetMongoDB)
    62. //登录
    63. // GET /flights/LAX-SFO
    64. app.Get("/login/:username/:pwd", handlers.ValidateUser)
    65. //注册
    66. app.Get("/login/:username/:pwd", handlers.SaveUser)
    67. // GET /api/register
    68. app.Get("/api/*", func(c *fiber.Ctx) error {
    69. msg := fmt.Sprintf("✋ %s", c.Params("*"))
    70. return c.SendString(msg) // => ✋ register
    71. })
    72. // GET /flights/LAX-SFO
    73. app.Get("/flights/:from-:to", func(c *fiber.Ctx) error {
    74. msg := fmt.Sprintf("💸 From: %s, To: %s", c.Params("from"), c.Params("to"))
    75. return c.SendString(msg) // => 💸 From: LAX, To: SFO
    76. })
    77. // GET /dictionary.txt
    78. app.Get("/:file.:ext", func(c *fiber.Ctx) error {
    79. msg := fmt.Sprintf("📃 %s.%s", c.Params("file"), c.Params("ext"))
    80. return c.SendString(msg) // => 📃 dictionary.txt
    81. })
    82. // GET /john/75
    83. app.Get("/:name/:age/:gender?", func(c *fiber.Ctx) error {
    84. msg := fmt.Sprintf("👴 %s is %s years old", c.Params("name"), c.Params("age"))
    85. return c.SendString(msg) // => 👴 john is 75 years old
    86. })
    87. // GET /john
    88. app.Get("/:name", func(c *fiber.Ctx) error {
    89. msg := fmt.Sprintf("Hello, %s 👋!", c.Params("name"))
    90. return c.SendString(msg) // => Hello john 👋!
    91. })
    92. app.Post("/post", func(c *fiber.Ctx) error {
    93. payload := struct {
    94. Name string `json:"name"`
    95. Email string `json:"email"`
    96. }{}
    97. if err := c.BodyParser(&payload); err != nil {
    98. return err
    99. }
    100. return c.JSON(payload)
    101. })
    102. app.Post("/post", func(c *fiber.Ctx) error {
    103. user := new(entities.User)
    104. if err := c.BodyParser(user); err != nil {
    105. fmt.Println("error = ",err)
    106. return c.SendStatus(200)
    107. }
    108. // getting user if no error
    109. fmt.Println("user = ", user)
    110. fmt.Println("user = ", user.Pwd)
    111. fmt.Println("user = ", user.Username)
    112. return c.SendString(user.Username)
    113. })
    114. // websocket 服务
    115. app.Use("/ws", func(c *fiber.Ctx) error {
    116. // IsWebSocketUpgrade returns true if the client
    117. // requested upgrade to the WebSocket protocol.
    118. if websocket.IsWebSocketUpgrade(c) {
    119. c.Locals("allowed", true)
    120. return c.Next()
    121. }
    122. return fiber.ErrUpgradeRequired
    123. })
    124. app.Get("/ws/:id", websocket.New(func(c *websocket.Conn) {
    125. // c.Locals is added to the *websocket.Conn
    126. log.Println(c.Locals("allowed")) // true
    127. log.Println(c.Params("id")) // 123
    128. log.Println(c.Query("v")) // 1.0
    129. log.Println(c.Cookies("session")) // ""
    130. // websocket.Conn bindings https://pkg.go.dev/github.com/fasthttp/websocket?tab=doc#pkg-index
    131. var (
    132. mt int
    133. msg []byte
    134. err error
    135. )
    136. for {
    137. if mt, msg, err = c.ReadMessage(); err != nil {
    138. log.Println("read:", err)
    139. break
    140. }
    141. log.Printf("recv: %s", msg)
    142. if err = c.WriteMessage(mt, msg); err != nil {
    143. log.Println("write:", err)
    144. break
    145. }
    146. }
    147. }))
    148. app.Get("/index.html", handlers.StaticHtml)
    149. app.Post("/get", func(c *fiber.Ctx) error {
    150. fmt.Println("c.get-appID", c.Get("X-Auth-AppId"), c.Get("X-Auth-Token"))
    151. return c.SendString("Hello, World!")
    152. })
    153. app.Get("/index.html", handlers.DlpDemoCtx)
    154. //打印屏幕输入数据
    155. app.Put("/put", handlers.Reader)
    156. data, _ := json.MarshalIndent(app.Stack(), "", " ")
    157. fmt.Println(string(data))
    158. // //执行xxl-job服务
    159. app.Use("/xxlJob", handlers.XXLJob)
    160. //延迟执行
    161. time.AfterFunc(time.Second * 5, Get)
    162. //监听端口
    163. log.Fatal(app.Listen(":3000"))
    164. }
    165. /**
    166. * gin框架
    167. */
    168. func ginInit(){
    169. r := gin.Default()
    170. // 匹配/users/xxx
    171. r.GET("/users/:name", func(c *gin.Context) {
    172. name := c.Param("name")
    173. c.String(http.StatusOK, "Hello %s\n", name)
    174. })
    175. // 带query参数
    176. r.GET("/books", func(c *gin.Context) {
    177. name := c.Query("name")
    178. c.String(http.StatusOK, "finding for book: %s\n", name)
    179. })
    180. // 表单数据
    181. r.POST("/register", func(c *gin.Context) {
    182. email := c.PostForm("email")
    183. password := c.DefaultPostForm("password", "123456")
    184. c.JSON(http.StatusOK, gin.H {
    185. "email": email,
    186. "password": password,
    187. })
    188. })
    189. // 分组
    190. defaultHandler := func(c *gin.Context) {
    191. c.JSON(http.StatusOK, gin.H{
    192. "path": c.FullPath(),
    193. })
    194. }
    195. // group: v1
    196. v1 := r.Group("/v1")
    197. {
    198. v1.GET("/posts", defaultHandler)
    199. v1.GET("/series", defaultHandler)
    200. }
    201. // group: v2
    202. v2 := r.Group("/v2")
    203. {
    204. v2.GET("/posts", defaultHandler)
    205. v2.GET("/series", defaultHandler)
    206. }
    207. // 模板渲染
    208. //r.LoadHTMLGlob("templates/*")
    209. r.GET("/students", func(c *gin.Context) {
    210. stus := []entities.Student{
    211. { Name: "Jack", Age: 20 },
    212. { Name: "David", Age: 18 },
    213. }
    214. c.HTML(http.StatusOK, "arr.tmpl", gin.H{
    215. "title": "Gin",
    216. "stuArr": stus,
    217. })
    218. })
    219. // listen and serve on 0.0.0.0:8080
    220. r.Run(":8080")
    221. }
    222. // 触发xxl-job服务方法
    223. func Get(){
    224. fmt.Println("开启延时执行")
    225. resp, err := http.Get("http://127.0.0.1:3000/xxlJob")
    226. if err != nil {
    227. fmt.Println(err)
    228. return
    229. }
    230. defer resp.Body.Close()
    231. body, err := ioutil.ReadAll(resp.Body)
    232. fmt.Println(string(body))
    233. fmt.Println(resp.StatusCode)
    234. if resp.StatusCode == 200 {
    235. fmt.Println("ok")
    236. }
    237. }
    1. package testing
    2. import (
    3. "fmt"
    4. "strings"
    5. "github.com/buckhx/gobert/tokenize"
    6. "github.com/buckhx/gobert/tokenize/vocab"
    7. tf "github.com/tensorflow/tensorflow/tensorflow/go"
    8. "github.com/aclements/go-gg/generic/slice"
    9. )
    10. var corpus = `深度学习的基础是机器学习中的分散表示(distributed representation)。分散表示假定观测值是由不同因
    11. 子相互作用生成。在此基础上,深度学习进一步假定这一相互作用的过程可分为多个层次,代表对观测值的多层抽象。不同
    12. 的层数和层的规模可用于不同程度的抽象。深度学习运用了这分层次抽象的思想,更高层次的概念从低层次的概念学习得到。
    13. 这一分层结构常常使用贪心算法逐层构建而成,并从中选取有助于机器学习的更有效的特征。不少深度学习算法都以无监督
    14. 学习的形式出现,因而这些算法能被应用于其他算法无法企及的无标签数据,这一类数据比有标签数据更丰富,也更容易获
    15. 得。这一点也为深度学习赢得了重要的优势。`
    16. var question = "深度学习的基础是什么?"
    17. /* 判断是否是英文字符 */
    18. func isAlpha(c byte) bool {
    19. return (c>=65 && c<=90) || (c>=97 && c<=122)
    20. }
    21. func main() {
    22. modelPath := "../../../nlp/albert_QA/outputs/saved-model"
    23. vocabPath := "../../../nlp/nlp_model/albert_zh_base/vocab_chinese.txt"
    24. voc, err := vocab.FromFile(vocabPath)
    25. if err != nil {
    26. panic(err)
    27. }
    28. m, err := tf.LoadSavedModel(modelPath, []string{"train"}, nil)
    29. if err != nil {
    30. panic(err)
    31. }
    32. tkz := tokenize.NewTokenizer(voc)
    33. ff := tokenize.FeatureFactory{Tokenizer: tkz, SeqLen: 512}
    34. // 拼接输入
    35. input_tokens := question + tokenize.SequenceSeparator + corpus
    36. // 获取 token 向量
    37. f := ff.Feature(input_tokens)
    38. tids, err := tf.NewTensor([][]int32{f.TokenIDs})
    39. if err != nil {
    40. panic(err)
    41. }
    42. new_mask := make([]float32, len(f.Mask))
    43. for i, v := range f.Mask {
    44. new_mask[i] = float32(v)
    45. }
    46. mask, err := tf.NewTensor([][]float32{new_mask})
    47. if err != nil {
    48. panic(err)
    49. }
    50. sids, err := tf.NewTensor([][]int32{f.TypeIDs})
    51. if err != nil {
    52. panic(err)
    53. }
    54. fmt.Println(f.Tokens)
    55. //fmt.Println(f.TokenIDs)
    56. //fmt.Println(f.Mask)
    57. //fmt.Println(f.TypeIDs)
    58. res, err := m.Session.Run(
    59. map[tf.Output]*tf.Tensor{
    60. m.Graph.Operation("input_ids").Output(0): tids,
    61. m.Graph.Operation("input_mask").Output(0): mask,
    62. m.Graph.Operation("segment_ids").Output(0): sids,
    63. },
    64. []tf.Output{
    65. m.Graph.Operation("finetune_mrc/Squeeze").Output(0),
    66. m.Graph.Operation("finetune_mrc/Squeeze_1").Output(0),
    67. },
    68. nil,
    69. )
    70. if err != nil {
    71. panic(err)
    72. }
    73. //fmt.Println("DataType", res[0].DataType())
    74. //fmt.Println("Shape", res[0].Shape())
    75. //fmt.Println("Value", res[0].Value().([][]float32))
    76. st := slice.ArgMax(res[0].Value().([][]float32)[0])
    77. ed := slice.ArgMax(res[1].Value().([][]float32)[0])
    78. fmt.Println(len(f.Tokens), st, ed)
    79. if ed<st{ // ed 小于 st 说明未找到答案
    80. st = 0
    81. ed = 0
    82. }
    83. //ans := strings.Join(f.Tokens[st:ed+1], "")
    84. // 处理token中的英文,例如: 'di', '##st', '##ri', '##bu', '##ted', 're', '##pr', '##ese', '##nt', '##ation',
    85. ans := ""
    86. for i:=st;i<ed+1;i++ {
    87. if len(f.Tokens[i])>0 && isAlpha(f.Tokens[i][0]){ // 英文开头,加空格
    88. ans += " "+f.Tokens[i]
    89. } else if strings.HasPrefix(f.Tokens[i], "##"){ // ##开头,是英文中段,去掉##
    90. ans += f.Tokens[i][2:]
    91. } else {
    92. ans += f.Tokens[i]
    93. }
    94. }
    95. if strings.HasPrefix(ans, "[CLS]") || strings.HasPrefix(ans, "[SEP]") {
    96. fmt.Println("未找到答案。")
    97. } else {
    98. fmt.Println("Question: ", question)
    99. fmt.Println("Answer: ", ans)
    100. }
    101. }
    1. package testing
    2. import (
    3. "strings"
    4. "github.com/buckhx/gobert/tokenize"
    5. "github.com/buckhx/gobert/tokenize/vocab"
    6. tf "github.com/tensorflow/tensorflow/tensorflow/go"
    7. "github.com/aclements/go-gg/generic/slice"
    8. )
    9. const (
    10. MaxSeqLength = 512
    11. )
    12. /* 训练好的模型权重 */
    13. var m *tf.SavedModel
    14. var voc vocab.Dict
    15. func InitModel(modelPath string, vocabPath string) {
    16. var err error
    17. voc, err = vocab.FromFile(vocabPath)
    18. if err != nil {
    19. panic(err)
    20. }
    21. m, err = tf.LoadSavedModel(modelPath, []string{"train"}, nil)
    22. if err != nil {
    23. panic(err)
    24. }
    25. }
    26. /* 判断是否是英文字符 */
    27. func isAlphaQa(c byte) bool {
    28. return (c>=65 && c<=90) || (c>=97 && c<=122)
    29. }
    30. func BertQA(corpus string, question string) (ans string, err error) {
    31. tkz := tokenize.NewTokenizer(voc)
    32. ff := tokenize.FeatureFactory{Tokenizer: tkz, SeqLen: MaxSeqLength}
    33. // 拼接输入
    34. input_tokens := question + tokenize.SequenceSeparator + corpus
    35. // 获取 token 向量
    36. f := ff.Feature(input_tokens)
    37. tids, err := tf.NewTensor([][]int32{f.TokenIDs})
    38. if err != nil {
    39. return ans, err
    40. }
    41. new_mask := make([]float32, len(f.Mask))
    42. for i, v := range f.Mask {
    43. new_mask[i] = float32(v)
    44. }
    45. mask, err := tf.NewTensor([][]float32{new_mask})
    46. if err != nil {
    47. return ans, err
    48. }
    49. sids, err := tf.NewTensor([][]int32{f.TypeIDs})
    50. if err != nil {
    51. return ans, err
    52. }
    53. res, err := m.Session.Run(
    54. map[tf.Output]*tf.Tensor{
    55. m.Graph.Operation("input_ids").Output(0): tids,
    56. m.Graph.Operation("input_mask").Output(0): mask,
    57. m.Graph.Operation("segment_ids").Output(0): sids,
    58. },
    59. []tf.Output{
    60. m.Graph.Operation("finetune_mrc/Squeeze").Output(0),
    61. m.Graph.Operation("finetune_mrc/Squeeze_1").Output(0),
    62. },
    63. nil,
    64. )
    65. if err != nil {
    66. return ans, err
    67. }
    68. st := slice.ArgMax(res[0].Value().([][]float32)[0])
    69. ed := slice.ArgMax(res[1].Value().([][]float32)[0])
    70. if ed<st{ // ed 小于 st 说明未找到答案
    71. st = ed
    72. }
    73. //ans = strings.Join(f.Tokens[st:ed+1], "")
    74. // 处理token中的英文,例如: 'di', '##st', '##ri', '##bu', '##ted', 're', '##pr', '##ese', '##nt', '##ation',
    75. ans = ""
    76. for i:=st;i<ed+1;i++ {
    77. if len(f.Tokens[i])>0 && isAlphaQa(f.Tokens[i][0]){ // 英文开头,加空格
    78. ans += " "+f.Tokens[i]
    79. } else if strings.HasPrefix(f.Tokens[i], "##"){ // ##开头,是英文中段,去掉##
    80. ans += f.Tokens[i][2:]
    81. } else {
    82. ans += f.Tokens[i]
    83. }
    84. }
    85. if strings.HasPrefix(ans, "[CLS]") || strings.HasPrefix(ans, "[SEP]") {
    86. return "", nil
    87. } else {
    88. return ans, nil // 找到答案
    89. }
    90. }
    1. package testing
    2. import (
    3. "C"
    4. "encoding/json"
    5. "fmt"
    6. "reflect"
    7. "strings"
    8. "sync"
    9. "testing"
    10. "unsafe"
    11. )
    12. type fakeService struct {
    13. }
    14. type Post struct {
    15. Name string
    16. Address string
    17. }
    18. type Service interface {
    19. ListPosts() ([]*Post, error)
    20. }
    21. func ListPosts(svc Service) ([]*Post, error) { return svc.ListPosts()}
    22. func newFakeService() Service {
    23. return &fakeService{}
    24. }
    25. func (s *fakeService) ListPosts() ([]*Post, error) {
    26. posts := make([] *Post, 0)
    27. posts = append(posts, &Post{
    28. Name: "colin",
    29. Address:"Shenzhen",
    30. })
    31. posts = append(posts, &Post{
    32. Name:"alex",
    33. Address:"beijing",
    34. })
    35. return posts, nil
    36. }
    37. func TestListPosts(t *testing.T){
    38. fake := newFakeService()
    39. if _, err := ListPosts(fake); err != nil {
    40. t.Fatal("list posts failed")
    41. }
    42. }
    43. type Bird interface {
    44. Fly()
    45. Type() string
    46. }
    47. type Canary struct {
    48. Name string
    49. }
    50. func (c *Canary)Fly() {
    51. fmt.Printf("我是%s,用黄色的翅膀飞\n", c.Name)
    52. }
    53. func (c *Canary) Type() string {
    54. return c.Name
    55. }
    56. type Crow struct {
    57. Name string
    58. }
    59. func (c *Crow) Fly() {
    60. fmt.Printf("我是%s,我用黑色的翅膀飞\n", c.Name)
    61. }
    62. func (c *Crow) Type() string {
    63. return c.Name
    64. }
    65. func LetFay(bird Bird) {
    66. fmt.Printf("Let %s \n", bird.Type())
    67. bird.Fly()
    68. }
    69. func testFay(t *testing.T) {
    70. LetFay(&Canary{Name:"jj"})
    71. LetFay(&Crow{Name:"gg"})
    72. C.puts(C.CString("Hello, Cgo\n"))
    73. cblob := C.blob{} // 在GO程序中创建的C对象,存储在Go的内存空间
    74. cblob.repeat_time = 0
    75. cblob.str = C.CString("Hello, World\n") // C.CString 会在C的内存空间申请一个C语言字符串对象,再将Go字符串拷贝到C字符串
    76. ret := C.SayHello(&cblob) // &cblob 取C语言对象cblob的地址
    77. fmt.Println("ret", ret)
    78. fmt.Println("repeat_time", cblob.repeat_time)
    79. C.free(unsafe.Pointer(cblob.str)) // C.CString 申请的C空间内存不会自动释放,需要显示调用C中的free释放
    80. }
    81. //问题很简单,使⽤ channel 来控制打印的进度。使⽤两个 channel ,来分别控制数字和
    82. //字⺟的打印序列, 数字打印完成后通过 channel 通知字⺟打印, 字⺟打印完成后通知数
    83. //字打印,然后周⽽复始的⼯作。
    84. func testNumber(t *testing.T){
    85. l,n:=make(chan bool), make(chan bool)
    86. wait := sync.WaitGroup{}
    87. go func() {
    88. i:= 1
    89. for {
    90. select {
    91. case <- n:
    92. fmt.Print(i)
    93. i++
    94. fmt.Print(i)
    95. i++
    96. l <- true
    97. break
    98. default:
    99. break
    100. }
    101. }
    102. }()
    103. wait.Add(1)
    104. go func(wait *sync.WaitGroup) {
    105. str := "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
    106. i:= 1
    107. for {
    108. select {
    109. case <- l:
    110. if i >= strings.Count(str, "")-1 {
    111. wait.Done()
    112. return
    113. }
    114. fmt.Print(str[i:i+1])
    115. i++
    116. if i >= strings.Count(str, "") {
    117. i = 0
    118. }
    119. fmt.Print(str[i:i+1])
    120. i++
    121. n <- true
    122. break
    123. default:
    124. break
    125. }
    126. }
    127. }(&wait)
    128. n <- true
    129. wait.Wait()
    130. }
    131. //第⼀个⽅法使⽤的是golang内置⽅法 strings.Count ,可以⽤来判断在⼀个字符串中包含
    132. //的另外⼀个字符串的数量。
    133. func isUniqueString(s string) bool {
    134. if strings.Count(s, "") > 3000 {
    135. return false
    136. }
    137. for _,v := range s{
    138. if v > 127 {
    139. return false
    140. }
    141. if strings.Count(s, string(v)) > 1 {
    142. return false
    143. }
    144. }
    145. return true
    146. }
    147. //第⼆个⽅法使⽤的是golang内置⽅法 strings.Index 和 strings.LastIndex ,⽤来判断指
    148. //定字符串在另外⼀个字符串的索引未知,分别是第⼀次发现位置和最后发现位置。
    149. func isUniqueString2(s string) bool {
    150. if strings.Count(s, "") > 3000 {
    151. return false
    152. }
    153. for k,v := range s{
    154. if v > 127 {
    155. return false
    156. }
    157. if strings.Index(s, string(v)) != k {
    158. return false
    159. }
    160. }
    161. return true
    162. }
    163. //func reverString2(s string) (string, bool) {
    164. // str := [] rune(s)
    165. // l := len(str)
    166. // if len > 5000 {
    167. // return s, false
    168. // }
    169. // for i := 0; i< len/2 ; i++ {
    170. // str[i], str[l-1-i] = str[l-1-i], str[i]
    171. // }
    172. // return string(str), true
    173. //}
    174. //在json的规范中,对于数字类型是不区分整形和浮点型的。
    175. //上游将id改为string传给下游
    176. //下游使用json.number类型来避免对float64的使用
    177. func dealStrTest(t *testing.T){
    178. var request = `{"id":7044144249855934983,"name":"demo"}`
    179. var test interface{} // 不指定反序列化的类型
    180. err := json.Unmarshal([]byte(request), &test)
    181. if err != nil {
    182. fmt.Println("error:", err)
    183. }
    184. obj := test.(map[string]interface{})
    185. dealStr, err := json.Marshal(test)
    186. if err != nil {
    187. fmt.Println("error:", err)
    188. }
    189. id := obj["id"]
    190. // 反序列化之后重新序列化打印
    191. fmt.Println(string(dealStr))
    192. fmt.Printf("%+v\n", reflect.TypeOf(id).Name())
    193. fmt.Printf("%+v\n", id.(float64))
    194. }
    195. //饿汉模式
    196. type singleton struct {}
    197. var ins *singleton = &singleton{}
    198. var mu sync.Mutex
    199. var once sync.Once
    200. func GetInsOr() *singleton {
    201. if ins == nil {
    202. mu.Lock()
    203. if ins == nil {
    204. ins = &singleton{}
    205. }
    206. mu.Unlock()
    207. }
    208. return ins
    209. }
    210. //推荐 使用once.Do可以确保 ins 实例全局只被创建一次,once.Do 函数还可以确保当同时有多个创建动作时,只有一个创建动作在被执行。
    211. func GetIns() *singleton {
    212. once.Do(func() {
    213. ins = &singleton{}
    214. })
    215. return ins
    216. }
    217. //工厂模式
    218. //type Person struct {
    219. // Name string
    220. // Age int
    221. //}
    222. //
    223. //func (p Person) Greet() {
    224. // fmt.Printf("Hi! My name is %s", p.Name)
    225. //}
    226. //
    227. //func NewPerson(name string, age int) *Person {
    228. // return &Person{
    229. // Name: name,
    230. // Age: age,
    231. // }
    232. //}
    233. //抽象工厂模式
    234. //type Person interface {
    235. // Greet()
    236. //}
    237. //
    238. //type person struct {
    239. // name string
    240. // age int
    241. //}
    242. //
    243. //func (p person) Greet() {
    244. // fmt.Printf("Hi! My name is %s", p.name)
    245. //}
    246. //
    247. //// Here, NewPerson returns an interface, and not the person struct itself
    248. //func NewPerson(name string, age int) Person {
    249. // return person{
    250. // name: name,
    251. // age: age,
    252. // }
    253. //}
    254. type Person struct {
    255. name string
    256. age int
    257. }
    258. func NewPersonFactory(age int) func(name string) Person {
    259. return func(name string) Person {
    260. return Person{
    261. name: name,
    262. age: age,
    263. }
    264. }
    265. }
    266. func NewTest(){
    267. newBaby := NewPersonFactory(1)
    268. _ = newBaby("john")
    269. newTeenager := NewPersonFactory(16)
    270. _ = newTeenager("jill")
    271. }
    272. //type Doer struct {
    273. // name string
    274. //}
    275. //
    276. //func QueryUser(doer Doer) error {
    277. // req, err := http.NewRequest("Get", "http://iam.api.marmotedu.com:8080/v1/secrets", nil)
    278. // if err != nil {
    279. // return err
    280. // }
    281. // _, err := doer.Do(req)
    282. // if err != nil {
    283. // return err
    284. // }
    285. //
    286. // return nil
    287. //}
    288. //
    289. //
    290. //func TestQueryUser(t *testing.T) {
    291. // doer := NewMockHTTPClient()
    292. // if err := QueryUser(doer); err != nil {
    293. // t.Errorf("QueryUser failed, err: %v", err)
    294. // }
    295. //}
    296. // 策略模式
    297. // 定义一个策略类
    298. type IStrategy interface {
    299. do(int, int) int
    300. }
    301. // 策略实现:加
    302. type add struct{}
    303. func (*add) do(a, b int) int {
    304. return a + b
    305. }
    306. // 策略实现:减
    307. type reduce struct{}
    308. func (*reduce) do(a, b int) int {
    309. return a - b
    310. }
    311. // 具体策略的执行者
    312. type Operator struct {
    313. strategy IStrategy
    314. }
    315. // 设置策略
    316. func (operator *Operator) setStrategy(strategy IStrategy) {
    317. operator.strategy = strategy
    318. }
    319. // 调用策略中的方法
    320. func (operator *Operator) calculate(a, b int) int {
    321. return operator.strategy.do(a, b)
    322. }
    1. package testing
    2. import (
    3. tf "github.com/tensorflow/tensorflow/tensorflow/go"
    4. "github.com/tensorflow/tensorflow/tensorflow/go/op"
    5. "fmt"
    6. )
    7. func testTf() {
    8. // Construct a graph with an operation that produces a string constant.
    9. s := op.NewScope()
    10. c := op.Const(s, "Hello from TensorFlow version " + tf.Version())
    11. graph, err := s.Finalize()
    12. if err != nil {
    13. panic(err)
    14. }
    15. // Execute the graph in a session.
    16. sess, err := tf.NewSession(graph, nil)
    17. if err != nil {
    18. panic(err)
    19. }
    20. output, err := sess.Run(nil, []tf.Output{c}, nil)
    21. if err != nil {
    22. panic(err)
    23. }
    24. fmt.Println(output[0].Value())
    25. }
    26. //go 调用 tensorflow模型
    27. //参考 https://blog.csdn.net/cdj0311/article/details/81773849
    28. // 参考 https://github.com/jack139/gotf
    29. func model() {
    30. // 句子最大长度
    31. const MAXLEN int = 20
    32. // 将文本转换为id序列,为了实验方便直接使用转换好的ID序列即可
    33. input_data := [1][MAXLEN]float32{{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 208.0, 659.0, 180.0, 408.0, 42.0, 547.0, 829.0, 285.0, 334.0, 42.0, 642.0, 81.0, 800.0}}
    34. tensor, err := tf.NewTensor(input_data)
    35. if err != nil {
    36. fmt.Printf("Error NewTensor: err: %s", err.Error())
    37. return
    38. }
    39. //读取模型
    40. model, err := tf.LoadSavedModel("cnnModel", []string{"myTag"}, nil)
    41. if err != nil {
    42. fmt.Printf("Error loading Saved Model: %s\n", err.Error())
    43. return
    44. }
    45. // 识别
    46. result, err := model.Session.Run(
    47. map[tf.Output]*tf.Tensor{
    48. // python版tensorflow/keras中定义的输入层input_layer
    49. model.Graph.Operation("input_layer").Output(0): tensor,
    50. },
    51. []tf.Output{
    52. // python版tensorflow/keras中定义的输出层output_layer
    53. model.Graph.Operation("output_layer/Softmax").Output(0),
    54. },
    55. nil,
    56. )
    57. if err != nil {
    58. fmt.Printf("Error running the session with input, err: %s ", err.Error())
    59. return
    60. }
    61. // 输出结果,interface{}格式
    62. fmt.Printf("Result value: %v", result[0].Value())
    63. }
    1. module github.com/programmerug/fibergorm
    2. go 1.16
    3. require (
    4. github.com/aclements/go-gg v0.0.0-20170323211221-abd1f791f5ee
    5. github.com/aliyun/aliyun-oss-go-sdk v2.2.0+incompatible
    6. github.com/andybalholm/brotli v1.0.4 // indirect
    7. github.com/apache/rocketmq-client-go/v2 v2.1.0
    8. github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
    9. github.com/blugelabs/bluge v0.1.9
    10. github.com/buckhx/gobert v0.0.0-20190731141805-265756fed232
    11. github.com/bytedance/godlp v1.2.15
    12. github.com/fasthttp/websocket v1.4.5 // indirect
    13. github.com/fatih/set v0.2.1
    14. github.com/fsnotify/fsnotify v1.5.1 // indirect
    15. github.com/gin-gonic/gin v1.6.3
    16. github.com/go-redis/redis/v8 v8.11.4
    17. github.com/go-stack/stack v1.8.1 // indirect
    18. github.com/gofiber/fiber/v2 v2.24.0
    19. github.com/gofiber/websocket/v2 v2.0.15
    20. github.com/google/go-querystring v1.1.0
    21. github.com/kardianos/service v1.2.0
    22. github.com/klauspost/compress v1.14.1 // indirect
    23. github.com/olivere/elastic/v7 v7.0.31
    24. github.com/pkg/errors v0.9.1
    25. github.com/pyroscope-io/pyroscope v0.7.2
    26. github.com/satori/go.uuid v1.2.0 // indirect
    27. github.com/sirupsen/logrus v1.8.1 // indirect
    28. github.com/tensorflow/tensorflow v1.15.4
    29. github.com/tidwall/gjson v1.12.1 // indirect
    30. github.com/xxl-job/xxl-job-executor-go v1.1.0
    31. github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
    32. go.mongodb.org/mongo-driver v1.8.2
    33. go.uber.org/atomic v1.9.0 // indirect
    34. golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
    35. golang.org/x/net v0.0.0-20220111093109-d55c255bac03 // indirect
    36. golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
    37. golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
    38. gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
    39. gorm.io/driver/mysql v1.2.3
    40. gorm.io/gorm v1.22.5
    41. gorm.io/hints v1.1.0
    42. )