C++
Q： Visual Studio 编译工程报错:error C4996: ‘strtok’: This function or variable may be unsafe. Consider using strtok_s instead
工程属性 —>C/C++ —> 预处理器 —> 预处理器定义,添加宏定义
_CRT_SECURE_NO_WARNINGS
Q： pthread.h中报错：C2011“timespec”:“struct”类型重定义
tim.h已定义，pthread.h添加如下代码：
#if !defined( PTHREAD_H )
#define PTHREAD_H
#define HAVE_STRUCT_TIMESPEC
Q：Visual Studio 工程编译dll,接口的定义
A：接口的头文件应如下声明：
#define LIB_API __declspec(dllexport) //导出函数
extern "C" {
LIB_API char* say();
}
Q：C++调用DLL
dll的创建
//test.h
#ifdef __cplusplus
extern "C" {
#endif
__declspec(dllexport) int add(int a,int b);
#ifdef __cplusplus
}
#endif // __cplusplus
//test.cpp
#include "test.h"
int add(int a,int b) {
return a + b;
}
编译上述工程为dll，即生成test.dll，并复制到调用该dll工程的编译目录
dll的使用
#include<Windows.h>
#includ<iostream>
typedef int (*padd)(int,int);
int main(int argc, char *argv[]){
HMODULE test = LoadLibrary("test.dll");
if (test != NULL){
padd add = (padd)GetProcAddress(test,"add");
if (add != NULL){
std::cout << add(1,0) << std::endl;
}
}
}
Q:cmake 简单编译命令
sudo tar -zxvf test
cd test
sudo mkdir build
cd build/
sudo cmake -DCMAKE_INSTALL_PREFIX=/home/test .. #安装目录
sudo make -j4 #编译
sudo make install #安装
sudo ldconfig #更新软连接
#预编译版本
sudo ln -s cmake /usr/bin/cmake
**Q: mac安装LLVM**
```bash
$ cd where-you-want-llvm-to-live
$ svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm
$ cd llvm/tools
$ svn co http://llvm.org/svn/llvm-project/cfe/trunk clang
$ cd llvm/tools/clang/tools
$ svn co http://llvm.org/svn/llvm-project/clang-tools-extra/trunk extra
$ cd llvm/projects
$ svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt
$ cd llvm
$ mkdir build
$ cd build
$ cmake -G "Unix Makefiles" ..
$ make
$ make install
Q:cJSON的使用
#include "cJSON.h"
cJSON *root = cJSON_CreateObject();
cJSON *bboxs = cJSON_CreateArray();
char *resultJson;
cJSON_AddNumberToObject(root, "status", 1);
cJSON_AddStringToObject(root, "msg", "detect ok");
cJSON_AddItemToObject(root, "bboxs", bboxs);
for (int i = 0; i < 2; i++){
cJSON *bbox;
cJSON_AddItemToArray(bboxs, bbox = cJSON_CreateObject());
cJSON_AddNumberToObject(bbox, "score", 99);
cJSON_AddNumberToObject(bbox, "x", 1);
cJSON_AddNumberToObject(bbox, "y", 2);
cJSON_AddNumberToObject(bbox, "width", 3);
cJSON_AddNumberToObject(bbox, "height", 4);
}
resultJson = cJSON_PrintUnformatted(result);
Q:double/float转字符串(保留精度)
const float* output = this->infer(data,len);
std::vector<float> vector(output, output + this->output_size);
std::stringstream ss;
ss << std::setprecision(16);
std::copy(vector.begin(), vector.end(), std::ostream_iterator<double>(ss, ","));
std::string values = ss.str();
values.pop_back();
cJSON_AddStringToObject(result, "values", values.c_str());
resultJson = cJSON_PrintUnformatted(result);
Q:c#调用dll
note:若dll为X64,则c#应编译x64,否则报错
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace test
{
class Program
{
[DllImport(@"facedetection.dll", EntryPoint = "facedetect")]
extern static string facedetect(string src);
static void Main(string[] args)
{
string result = facedetect("test.jpg");
Console.WriteLine(result);
Console.ReadKey();
}
}
}
Q: c++时间差计算
#include <chrono>
auto start = std::chrono::steady_clock::now();
//do something
auto end = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed = end - start;
std::cout << "elapsed:" << elapsed.count() <<"s"<< std::endl;
Q: c++读写文件
#include <fstream>
#include <iomanip>
//写文件
ofstream out_txt_file;
out_txt_file.open("image.bat", ios::out | ios::trunc);
for (int i = 0; i < image.size(); i++) {
out_txt_file << to_string(image[i]) << endl;
}
out_txt_file.close();
//读文件
std::vector<float> image;
ifstream infile;
infile.open("image.bat");
assert(infile.is_open());
string s;
while(getline(infile,s)){
image.push_back(stof(s));
}
infile.close();
std::pair<char*, int> readFile(std::string path) {
fstream in(path, ios::in | ios::binary);
if (!in.is_open()) {
cerr << "file not find" << endl;
}
in.seekg(0, ios::end);
long size;
size = in.tellg();
char* buf = new char[size];
in.seekg(ios::beg);
in.read(buf, size);
return std::make_pair(buf, size);
}
Q: mmap
void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
- start：指定映射的内存地址,设置为 NULL ->操作系统自动选择合适的内存地址
- length：映射地址空间的字节数,从被映射文件开头 offset 个字节开始统计
- prot：指定共享内存的访问权限,PROT_READ(可读),PROT_WRITE(可写), PROT_EXEC(可执行), PROT_NONE(不可访问)
- flags：MAP_SHARED(共享的), MAP_PRIVATE(私有)
- fd：映射的文件句柄,fd = open(file_path, flag, mode);
- offset：映射文件的偏移量,设置0 ->从文件头部开始映射
int32_t fd = open(filePath.c_str(), O_RDONLY, 0);
if(fd < 0){
return nullptr;
}
char* pBuffer = (char)mmap(nullptr, (size_t)fileSize, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0);
Q:cmake添加编译选项
SET(CMAKE_C_FLAGS_DEBUG "-g -D_DEBUG -O0")
SET(CMAKE_C_FLAGS_RELEASE "-g -DNDEBUG -O3")
SET(CMAKE_CXX_FLAGS_DEBUG "-g -D_DEBUG -O0")
SET(CMAKE_CXX_FLAGS_RELEASE "-g -DNDEBUG -O3")
Q:python call c++ (vector)
#include <vector>
#include "TTSEngine.h"
std::shared_ptr<TTSEngine> tts(new TTSEngine());
extern "C" {
void infer(std::vector<float>* output,int* data,int len){
const float* net_output = tts->infer(data,len);
for (int i = 0; i < tts->output_size; ++i) {
output->push_back(net_output[i]);
}
}
/* python vector bind */
std::vector<float>* new_vector(){
return new std::vector<float>;
}
void delete_vector(std::vector<float>* v){
delete v;
}
int vector_size(std::vector<float>* v){
return v->size();
}
float vector_get(std::vector<float>* v, int i){
return v->at(i);
}
}
ttslib = cdll.LoadLibrary("libtts.dylib")
class Vector(object):
ttslib.new_vector.restype = c_void_p
ttslib.new_vector.argtypes = []
ttslib.delete_vector.restype = None
ttslib.delete_vector.argtypes = [c_void_p]
ttslib.vector_size.restype = c_int
ttslib.vector_size.argtypes = [c_void_p]
ttslib.vector_get.restype = c_float
ttslib.vector_get.argtypes = [c_void_p, c_int]
def __init__(self):
self.vector = ttslib.new_vector()
def __del__(self):
ttslib.delete_vector(self.vector)
def __len__(self):
return ttslib.vector_size(self.vector)
def __getitem__(self, i):
if 0 <= i < len(self):
return ttslib.vector_get(self.vector, c_int(i))
raise IndexError('Vector index out of range')
def __repr__(self):
return '[{}]'.format(', '.join(str(self[i]) for i in range(len(self))))
def getInferOutput(self,input,seq_len):
ttslib.infer(self.vector,input,c_int(seq_len))
if __name__ == '__main__':
result = Vector()
input = [0,1,2,3]
seq_len = len(input)
input = (c_int * seq_len)(*input)
result.getInferOutput(input,seq_len)
return np.array(result)
异步方法 超时机制
#include <thread>
#include <future>
std::promise<bool> promise;
std::thread([&] {
bool status = false;
try {
//do something
} catch (...) {
status = false;
}
promise.set_value(status);
}).detach();
auto future = promise.get_future();
if(future.wait_for(std::chrono::milliseconds(1)) == std::future_status::timeout) {
return false;
}
auto result = future.get();
return result;
Q：c++11 std::string/std::wstring
#include <string>
#include <locale>
#include <codecvt>
// convert string to wstring
inline std::wstring to_wide_string(const std::string& input){
std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
return converter.from_bytes(input);
}
// convert wstring to string
inline std::string to_byte_string(const std::wstring& input){
//std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
return converter.to_bytes(input);
}
/*
1.The first 128 characters (US-ASCII) in UTF-8 format only need one byte.
2. The next 1,920 characters need two bytes to encode,which covers the remainder of almost all Latin-script alphabets.
3. Three bytes are needed for characters in the rest of the Basic Multilingual Plane, which contains virtually all characters in common use, including most Chinese, Japanese and Korean characters.
4. Four bytes are needed for characters in the other planes of Unicode,which include less common CJK characters, various historic scripts,mathematical symbols, and emoji (pictographic symbols).
*/
inline int getCharLength(char letter) {
int bytes = 1;
if ((letter & 0x80) == 0x00) bytes = 1 ;
else if ((letter & 0xE0) == 0xC0) bytes = 2;
else if ((letter & 0xF0) == 0xE0) bytes = 3;
else if ((letter & 0xF8) == 0xF0) bytes = 4;
return bytes;
}
inline void string2chars(const std::string& text, std::vector<std::string>& subs) {
subs.clear();
int num_bytes = 1;
for (size_t i = 0; i < text.length(); i += num_bytes) {
num_bytes = getCharLength(text[i]);
subs.push_back(text.substr(i, num_bytes));
}
}
Q：c++分割字符串
inline void split(const std::string &text,
const std::string &delimiter,
std::vector<std::string> &tokens){
std::string::size_type last_pos = text.find_first_not_of(delimiter,0);
std::string::size_type pos = text.find_first_of(delimiter,last_pos);
while (std::string::npos != pos || std::string::npos != last_pos){
tokens.push_back(text.substr(last_pos,pos - last_pos));
last_pos = text.find_first_not_of(delimiter,pos);
pos = text.find_first_of(delimiter,last_pos);
}
}
Q：c++ 多线程条件变量通信
#include <thread>
#include <mutex>
#include <deque>
#include <condition_variable>
std::mutex mtx; // 互斥量,保护共享资源
std::condition_variable not_empty; // 条件变量,同步线程
std::deque<int> shared_data = {1,2,3}; // 共享数据
int takeData(){
std::unique_lock<std::mutex> lck(mtx);
while (shared_data.empty()) {
not_empty.wait(lck);
}
if (!shared_data.empty()) {
int data = shared_data.front();
shared_data.pop_front();
return data;
}
return 0;
}
void pushData(int data){
std::unique_lock<std::mutex> lck(mtx);
shared_data.emplace_back(data);
not_empty.notify_all();
}
void worker_thread(int id) {
int data = takeData();
if(data > 0){
std::cout << id << " data:" << data << std::endl;
pushData(data);
} else{
std::cout << "======" << id << "====" << std::endl;
}
}
int main(){
std::thread t1(worker_thread,0);
std::thread t2(worker_thread,1);
std::thread t3(worker_thread,2);
std::thread t4(worker_thread,3);
std::thread t5(worker_thread,4);
std::thread t6(worker_thread,5);
t1.join();
t2.join();
t3.join();
t4.join();
t5.join();
t6.join();
return 0;
}
Q：c++ 跨平台判断
- Windows
WIN32、_WIN32、_WIN32_、WIN64、_WIN64、_WIN64_ - Android
ANDROID、_ANDROID_ - Linux
__linux__ - iOS/Mac
__APPLE__ TARGET_OS_IPHONE TARGET_IPHONE_SIMULATOR TARGET_OS_MAC
#if defined(TARGET_OS_MAC) || defined(__APPLE__)
/*code*/
#endif
**Q：c++ 遍历UTF-8字符串
inline int getcharBytes(char letter) {
int bytes = 1;
if ((letter & 0x80) == 0x00) bytes = 1 ;
else if ((letter & 0xE0) == 0xC0) bytes = 2;
else if ((letter & 0xF0) == 0xE0) bytes = 3;
else if ((letter & 0xF8) == 0xF0) bytes = 4;
return bytes;
}
inline void convertTextToChars(const std::string& text, std::vector<std::string>& subs) {
subs.clear();
int bytes = 1;
for (size_t i = 0; i < text.length(); i += bytes) {
bytes = getcharBytes(text[i]);
subs.push_back(text.substr(i, bytes));
}
}
Python
Q：python打包发布
1.项目根目录新建setup.py
from distutils.core import setup
setup(
name='test', #打包的输出文件名
version='1.0.0',
py_modules=['test'] #打包的.py文件
)
依赖文件 requirements.txt生成
pip freeze > requirements.txt
opencv==3.4.2
numpy==1.16.4
scikit-image==0.15.0
或者 conda list -e > requirements.txt
numpy=1.16.4
opencv=3.4.2
scikit-image=0.15.0
2.项目根目录下执行命令
sudo python setup.py sdist
生成dist目录且包含test-1.0.0.tar.gz分发包
3.安装：解压分发包并定位到解压目录后执行安装命令
pip install -r requirements.txt #安装依赖 或者 conda install --yes --file requirements.txt
sudo python setup.py install
4.使用
import test
test.sayHello();//#调用模块的sayHello方法
Q:jupyter notebook 运行于指定的conda环境上
$ conda install nb_conda #重启jupyter notebook 选择kernel即可
$ conda info --envs
# conda environments:
#
base /Volumes/storage/mac/miniconda3
python36 * /Volumes/storage/mac/miniconda3/envs/python36
$ jupyter kernelspec list
Available kernels:
reco_base /Users/jiaopan/Library/Jupyter/kernels/reco_base
python3 /Volumes/storage/mac/miniconda3/envs/python36/share/jupyter/kernels/python3
$ cd /Volumes/storage/mac/miniconda3/envs/python36
$ vi kernel.json
{
"argv": [
"/Volumes/storage/mac/miniconda3/envs/python36/bin/python",#修改
"-m",
"ipykernel_launcher",
"-f",
"{connection_file}"
],
"display_name": "Python 3",
"language": "python"
}
Q:设置jupyter notebook可远程访问
$ jupyter notebook --generate-config
Writing default config to: /home/adminpc/.jupyter/jupyter_notebook_config.py
$ jupyter notebook password
Enter password:
Verify password:
[NotebookPasswordApp] Wrote hashed password to /home/adminpc/.jupyter/jupyter_notebook_config.json
$ vim /home/adminpc/.jupyter/jupyter_notebook_config.py
c.NotebookApp.ip='*'
# 密钥位于adminpc/.jupyter/jupyter_notebook_config.json
c.NotebookApp.password = 'sha1:0f0fe326cb63:0e48570b0b0952dfe72fefb7bd012934968ea428'
c.NotebookApp.open_browser = False
c.NotebookApp.port =8888
$ jupyter notebook
[I 14:14:05.933 NotebookApp] Serving notebooks from local directory: /home/adminpc/jiaopan/projects
[I 14:14:05.933 NotebookApp] The Jupyter Notebook is running at:
[I 14:14:05.933 NotebookApp] http://admin-pc:8888/
[I 14:14:05.933 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).
时间格式转换
print(datetime.strftime(datetime.now(),"%Y-%m-%d %H:%M:%S"))
print(datetime.strptime("2021-03-15 02:26:07","%Y-%m-%d %H:%M:%S"))
#时间戳转换
timeArray = time.localtime(int(time.time()))
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
print(otherStyleTime)
Q:调用c++库文件
so = ctypes.cdll.LoadLibrary("/home/jiaopan/projects/c++/detector-onnx-linux/cmake-build-debug/libdetector.so")
model_path = bytes("/home/jiaopan/projects/c++/detector-onnx-linux/model/yolov5s.onnx", "utf-8")
status = so.init(model_path, 1)
print(status)
image = bytes("/home/jiaopan/Downloads/bus.jpg", "utf-8")
detectFile = so.detectByFile
detectFile.restype = ctypes.c_char_p
result = detectFile(image, ctypes.c_float(0.5))
result = ctypes.string_at(result, -1).decode("utf-8")
print(result)
Q:pytorch2paddle
# -*- coding: utf-8 -*-
# @Time : 2022/3/8 14:38
# @Author : jiaopaner
import sys
sys.path.insert(0, './')
import torch
import numpy as np
from torchvision.models import AlexNet
from torch.utils.model_zoo import load_url as load_state_dict_from_url
from collections import OrderedDict
from craft import CRAFT
def copyStateDict(state_dict):
if list(state_dict.keys())[0].startswith("module"):
start_idx = 1
else:
start_idx = 0
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = ".".join(k.split(".")[start_idx:])
new_state_dict[name] = v
return new_state_dict
if __name__ == '__main__':
# 构建输入
input_data = np.random.rand(1, 3, 960, 960).astype("float32")
# 获取PyTorch Module
# torch_module = AlexNet()
# torch_state_dict = load_state_dict_from_url('https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth')
# torch_module.load_state_dict(torch_state_dict)
# # 设置为eval模式
# torch_module.eval()
net = CRAFT()
net.load_state_dict(copyStateDict(torch.load("/Volumes/storage/resources/models/paddle-ocr-models/craft_mlt_25k.pth", map_location="cpu")))
net.eval()
#dynamic shape
x = torch.randn((1, 3, 960, 960))
torch.onnx.export(net, x, './pd_model/model.onnx', opset_version=11, input_names=["input"],
output_names=["output"], dynamic_axes={'input': [2,3]})
# 进行转换
# from x2paddle.convert import pytorch2paddle
#
# pytorch2paddle(net,
# save_dir="pd_model_trace",
# jit_type="trace",
# input_examples=[torch.tensor(input_data)])
x2paddle --framework=onnx --model=./pd_model/model.onnx --save_dir=pd_model_dynamic
####x2paddle_code.py####
if __name__ == '__main__':
x = paddle.randn((1,3,100,100),dtype=paddle.float32)
result = main(x)
print(result)
Q:multiprocessing.Queue.qsize() error in mac
class SharedCounter(object):
""" A synchronized shared counter.
The locking done by multiprocessing.Value ensures that only a single
process or thread may read or write the in-memory ctypes object. However,
in order to do n += 1, Python performs a read followed by a write, so a
second process may read the old value before the new one is written by the
first process. The solution is to use a multiprocessing.Lock to guarantee
the atomicity of the modifications to Value.
This class comes almost entirely from Eli Bendersky's blog:
http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
"""
def __init__(self, n = 0):
self.count = multiprocessing.Value('i', n)
def increment(self, n = 1):
""" Increment the counter by n (default = 1) """
with self.count.get_lock():
self.count.value += n
@property
def value(self):
""" Return the value of the counter """
return self.count.value
class Queue(multiprocessing.queues.Queue):
""" A portable implementation of multiprocessing.Queue.
Because of multithreading / multiprocessing semantics, Queue.qsize() may
raise the NotImplementedError exception on Unix platforms like Mac OS X
where sem_getvalue() is not implemented. This subclass addresses this
problem by using a synchronized shared counter (initialized to zero) and
increasing / decreasing its value every time the put() and get() methods
are called, respectively. This not only prevents NotImplementedError from
being raised, but also allows us to implement a reliable version of both
qsize() and empty().
"""
def __init__(self, *args, **kwargs):
super(Queue, self).__init__(*args, **kwargs)
self.size = SharedCounter(0)
def put(self, *args, **kwargs):
self.size.increment(1)
super(Queue, self).put(*args, **kwargs)
def get(self, *args, **kwargs):
self.size.increment(-1)
return super(Queue, self).get(*args, **kwargs)
def qsize(self):
""" Reliable implementation of multiprocessing.Queue.qsize() """
return self.size.value
def empty(self):
""" Reliable implementation of multiprocessing.Queue.empty() """
return not self.qsize()
opencv
Q:cv::Mat替换值
dst = src.clone();
cv::Mat Roi(dst, cv::Rect(x, y, width, height));
cv::RNG rnger(cv::getTickCount());
rnger.fill(Roi, cv::RNG::UNIFORM, cv::Scalar::all(0), cv::Scalar::all(256));
//创建图像
Mat img(Size(320,240), CV_8UC3);
//选择ROI
Mat roi(img, Rect(10, 10, 100, 100));
//将ROI的颜色填充为绿色,原始的img图像将会改变
roi = Scalar(0,255,0);
//setTo
cv::Mat image(8, 9, CV_8UC3);
image.setTo(255); //将值全部设置成255
cv::Rect roi(1, 1, 4, 3);
image(roi).setTo(200); //指定像素点区域的值都设置成200
image.at<cv::Vec3b>(0,0)[1] = 99;
image.setTo(88,image == 99); //将M中等于99的值设置成88
//参数2是条件判断 可以是小于 大于等等
cv::Mat mask = cv::Mat::zeros(image.size(),CV_8UC3);
//建立与原图一样大小的mask图像，并将所有像素初始化为0
mask(roi).setTo(1);
mask.at<cv::Vec3b>(0,0)[1] = 66;
image.setTo(55, mask); //在矩阵M中，把矩阵mask中不为0的值全部变为55
Q:convertTo
void cv::Mat::convertTo(OutputArray m, int rtype, double alpha = 1, double beta = 0) const
image.convertTo(dst,CV_8UC1);//类型转换 CV_32FC1->CV_8UC1
image.convertTo(dst, CV_8CU1, 1.0/255, 0.5);
/*
alpha,beta-> image = (image->element * alpha)->element + beta
*/
Q:connectedComponentsWithStats
//像素相同的区域连通函数
//image:单通道
//4连通:上下左右,8连通:上下左右+左上、右上、右下、左下
cv::Mat labels,stats,centroids;
int n_comps = cv::connectedComponentsWithStats(image,labels,stats,centroids,4);
//n_comps:连通区域的数量。
//labels:labels.shape = image.shape,每一个连通区域会有一个唯一标识,标识从0索引,0为背景
//stats:包含5个参数(x,y,h,w,s),对应每一个连通区域外接矩形的起始坐标x,y;外接矩形的width,height;s为labels对应的连通区域的像素个数
//centroids : 返回的是连通区域的质心
Q:cv2::threshold
cv::threshold (src,dst,thresh, maxval, type)
src:源矩阵,单通道
dst:目标
thresh:阈值,取值范围 0～255
maxval:填充色,取值范围 0～255
type：阈值类型
阈值 小于阈值的像素点 大于阈值的像素点
0 置0 置填充色
1 置填充色 置0
2 保持原色 置灰色
3 置0 保持原色
4 保持原色 置0
Q:读取摄像头/视频流
cap = cv2.VideoCapture("/home/jiaopan/Downloads/jd2.mp4")
index = 0;
detect = so.detectByBase64
detect.restype = ctypes.c_char_p
if cap.isOpened():
while True:
ret, frame = cap.read()
#frame = cv2.resize(frame, (320, 320), interpolation=cv2.INTER_AREA)
if index % 1 == 0:
start = datetime.now()
#cv2.imwrite("frame.jpg",frame)
#result = detectFile(bytes("frame.jpg","utf-8"), ctypes.c_float(0.6))
image = frame2base64(frame)
result = detect(image, ctypes.c_float(0.4))
end = datetime.now()
print("time cost:",(end-start).total_seconds())
result = ctypes.string_at(result, -1).decode("utf-8")
result = json.loads(result)
data = result["data"]
print(data)
for box in data:
x = int(box["location"]["x"])
y = int(box["location"]["y"])
width = int(box["location"]["width"])
height = int(box["location"]["height"])
cv2.putText(frame, box["label"], (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 255), 1)
cv2.rectangle(frame,(x,y),(x+width,y+height),(255,0,0),2)
cv2.imshow('image', frame)
k = cv2.waitKey(20)
index += 1
# q键退出
if (k & 0xff == ord('q')):
break
cap.release()
cv2.destroyAllWindows()
Q:图片转base64编码/base64编码还原图片
def image_to_base64(image):
image = cv2.imencode('.jpg', image)[1]
image_code = str(base64.b64encode(image))[2:-1]
return image_code
image = cv2.imread("test-04-output-0001.jpg")
print(image_to_base64(image))
def base64_to_image(base64_code):
# base64解码
img_data = base64.b64decode(base64_code)
# 转换为np数组
img_array = np.fromstring(img_data, np.uint8)
# 转换成opencv可用格式
img = cv2.imdecode(img_array, cv2.COLOR_RGB2BGR)
return img
#include <iostream>
#include <fstream>
#include <sstream>
#include <map>
#include <opencv2/opencv.hpp>
const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
bool isBase64( unsigned char c ){
return (isalnum(c) || (c == '+') || (c == '/'));
}
std::string base64Decode(std::string const& encoded_string){
int in_len = encoded_string.size();
int i = 0;
int j = 0;
int index = 0;
unsigned char char_array_4[4], char_array_3[3];
std::string ret;
while (in_len-- && (encoded_string[index] != '=') && isBase64(encoded_string[index])){
char_array_4[i++] = encoded_string[index]; index++;
if (i == 4){
for (i = 0; i < 4; i++){
char_array_4[i] = base64_chars.find(char_array_4[i]);
}
char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
for (i = 0; (i < 3); i++){
ret += char_array_3[i];
}
i = 0;
}
}
if (i){
for (j = i; j < 4; j++) char_array_4[j] = 0;
for (j = 0; j < 4; j++) char_array_4[j] = base64_chars.find(char_array_4[j]);
char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
for (j = 0; (j < i - 1); j++) ret += char_array_3[j];
}
return ret;
}
cv::Mat base64ToMat(std::string &base64_data){
std::string decoded_string = base64Decode(base64_data);
std::vector<uchar> data(decoded_string.begin(), decoded_string.end());
cv::Mat image = imdecode(data, cv::IMREAD_UNCHANGED);
return image;
}
Q：opencv Mat和byte互转
unsigned char* matToBytes(Mat image){
int size = image.total() * image.elemSize();
unsigned char* bytes = new unsigned char[size]; // you will have to delete[] that later
std::memcpy(bytes, image.data, size * sizeof(unsigned char));
return bytes;
}
Mat bytesToMat(unsigned char* bytes,int width,int height){
Mat image = Mat(height,width,CV_8UC3,bytes).clone(); // make a copy
return image;
}
Q:opencv Mat 转std::vector std::vector转std::string
cv::Mat output = output.reshape(1, 1);
std::vector<double> vetor = (std::vector<double>)output;
std::stringstream ss;
ss << std::setprecision(16);//精度设置
std::copy(vector.begin(), vector.end(), std::ostream_iterator<double>(ss, ","));
std::string values = ss.str();
values.pop_back();//去掉末尾多余的分隔符
std::cout << values;
Mat convertToMat(std::string str) {
std::vector<double> v;
std::stringstream ss(str);
ss << std::setprecision(16);
std::string token;
while (std::getline(ss, token, ',')) {
v.push_back(std::stod(token));
}
Mat output = cv::Mat(v, true).reshape(1, 1);
return output;
}
cv::FileStorage fs("../features.xml", cv::FileStorage::WRITE);
fs << "features" << output;
fs.release();
Q: opencv 截取图片的矩形区域/不规则图像
#矩形
cv::rectangle(image, Rect(x, y, w, h), Scalar(0, 255, 0), 2);//绘制矩形框
Mat dst = image(cv::Range(y,y+h),cv::Range(x,x+w));
#不规则区域
image = cv2.imread("test.jpg")
points = np.array([[[101, 116], [315, 18], [335, 64], [121, 161]]])
height,width,_ = image.shape
mask = np.zeros((height,width), np.uint8)
cv2.polylines(mask, points, 1, 255)
cv2.fillPoly(mask, points, 255)
dst = cv2.bitwise_and(image, image, mask=mask)
#设置白色背景
# bg = np.ones_like(image, np.uint8) * 255
# cv2.bitwise_not(bg, bg, mask=mask)
# dst_white = bg + dst
cv2.imwrite("roi.jpg", dst)
Q: Mat直接赋值
double left = 0,right = 0.5,top = 0,bottom = 0.5;
cv::Mat pbox = (cv::Mat_<double>(4,2)<<left,top,right,top,right,bottom,left,bottom);
cv::Mat data_map(height, width, CV_8UC1, (unsigned char *)vec.data());
cv::Mat data_map(height, width, CV_32FC1, (float *)vec.data());
Q: Mat 行列 SUM/MAX/MIN/AVG
cv::reduce(pbox,sum_value,1,cv::REDUCE_SUM);
Q: 仿射变换
import cv2
import numpy as np
def perspective_transform_image(image, points):
rect_xmin, _, _, rect_xmax = np.sort(points[:, :, 0]).flatten()
rect_ymin, _, _, rect_ymax = np.sort(points[:, :, 1]).flatten()
image = image[rect_ymin:rect_ymax, rect_xmin:rect_xmax]
points[:, :, 0] = points[:, :, 0] - np.array(rect_xmin)
points[:, :, 1] = points[:, :, 1] - np.array(rect_ymin)
height, width, _ = image.shape
mask = np.zeros((height, width), np.uint8)
cv2.fillPoly(mask, points, (255, 255, 255))
dst = cv2.bitwise_and(image, image, mask=mask)
# 设置白色背景
# bg = np.ones_like(image, np.uint8) * 255 # cv2.bitwise_not(bg, bg, mask=mask) # dst_white = bg + dst
name = str(time.time())
cv2.imwrite("../inference_results/%s_roi.jpg" % (name), dst)
left_top, right_top, right_bottom, left_bottom = np.squeeze(points, 0)
left_top_x, left_top_y = left_top
left_bottom_x, left_bottom_y = left_bottom
roi_height = int(
math.sqrt(math.pow(abs(left_bottom_x - left_top_x), 2) + math.pow(abs(left_bottom_y - left_top_y), 2)))
roi_width = int(math.sqrt(math.pow(height, 2) + math.pow(width, 2)))
source_points = np.float32([left_top, right_top, left_bottom, right_bottom])
target_points = np.float32([[0, 0], [roi_width, 0], [0, roi_height], [roi_width, roi_height], ])
mat = cv2.getPerspectiveTransform(source_points, target_points)
dst = cv2.warpPerspective(dst, mat, (roi_width, roi_height))
cv2.imwrite("../inference_results/%s_dst.jpg" % (name), dst)
return dst
def affine_transform(image, angle):
height, width, _ = image.shape
center = (width // 2, height // 2)
matrx = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(image, matrx, (width, height), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
cv2.imwrite("../inference_results/rotated.jpg", rotated)
x, y = center
print(x, y)
# xmin = 243
# ymin = 215 xmin = 100
ymin = 21
# points = np.array([[243, 236], [265, 215], [400, 348], [378, 369]])
points = np.array([[100, 119], [317, 21], [334, 60], [118, 159]])
points[:, 0] = points[:, 0] - xmin + 18
points[:, 1] = points[:, 1] - ymin
angle = math.radians(angle)
beta = round(math.sin(angle), 2)
alpha = round(math.cos(angle), 2)
print(beta, alpha)
x_points = points[:, 0].copy() - x
y_points = -(points[:, 1].copy() - y)
print("origin points:", x_points, y_points)
points[:, 0] = x_points * alpha - y_points * beta + x
print(points)
print(y_points * alpha + x_points * beta)
points[:, 1] = y - (y_points * alpha + x_points * beta)
print(points)
vis = rotated.copy()
cv2.drawContours(vis, [points], 0, (0, 0, 255), 2)
cv2.imwrite("../inference_results/vis.jpg", vis)
Java
Q：springboot切面编程实例
@Aspect
@Component
@Slf4j
public class WebLogAspect {
//定义一个切入点，对该包下的所有函数方法执行前，执行@Before()
@Pointcut("execution(public * com.jp.controller..*.*(..))")
public void webLog(){}
@Before("webLog()")
public void doBefore(JoinPoint joinPoint) throws Throwable {
// 接收到请求，记录请求内容
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
HttpServletRequest request = attributes.getRequest();
// 记录下请求内容
log.info("URL : " + request.getRequestURL().toString());
log.info("HTTP_METHOD : " + request.getMethod());
log.info("IP : " + request.getRemoteAddr());
log.info("CLASS_METHOD : " + joinPoint.getSignature().getDeclaringTypeName() + "." + joinPoint.getSignature().getName());
log.info("ARGS : " + Arrays.toString(joinPoint.getArgs()));
}
@AfterReturning(returning = "ret", pointcut = "webLog()")
public void doAfterReturning(Object ret) throws Throwable {
// 处理完请求，返回内容
log.info("RESPONSE : " + ret);
}
}
Q：Mac，MavenReportException: Error while creating archive: Unable to find javadoc command: The environment variable JAVA_HOME is not correctly set
在pom.xml文件中添加javadoc命令的位置
<properties>
<javadocExecutable>/Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/bin/javadoc</javadocExecutable>
</properties>
Q：maven引入本地依赖
<dependency>
<groupId>com.oracle</groupId>
<artifactId>ojdbc6</artifactId>
<scope>system</scope>
<version>1.0</version>
<systemPath>${basedir}/lib/ojdbc6-11.2.0.1.0.jar</systemPath>
</dependency>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<includeSystemScope>true</includeSystemScope>
<mainClass>mainclass</mainClass>
</configuration>
</plugin>
</plugins>
</build>
Q：普通maven项目打包配置
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId> maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>jiaopaner.spark.learning.App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
Q:springboot-kafka集群
# 显示启动
./bin/kafka-server-start.sh ./config/server.properties
# 在后台启动
./bin/kafka-server-start.sh -daemon ./config/server.properties
#创建topic
# --create：表示创建
# --zookeeper 后面的参数是zk的集群节点
# --replication-factor 3 ：表示复本数
# --partitions 3：表示分区数
# --topic test：表示topic的主题名称
./bin/kafka-topics.sh --create --zookeeper server01:2181,server02:2181,server03:2181 --replication-factor 3 --partitions 3 --topic test
#查看topic
./bin/kafka-topics.sh --list --zookeeper server01:2181
#删除topic
./bin/kafka-topics.sh --delete --zookeeper server01:2181 --topic test
#生产者命令
/bin/kafka-console-producer.sh --broker-list server01:9092 --topic test
#消费者命令
/bin/kafka-console-consumer.sh --zookeeper server01:2181 --from-beginning --topic test
Q:maven dependency scope设置
- provided:编译和测试阶段依赖
<dependency>
<groupId>org.jiaoper/groupId>
<artifactId>utils-common</artifactId>
<version>3.0-alpha-1</version>
<scope>provided</scope>
</dependency>
- test:测试阶段依赖
<dependency>
<groupId>org.jiaoper/groupId>
<artifactId>utils-common</artifactId>
<version>3.0-alpha-1</version>
<scope>test</scope>
</dependency>
- compile:编译、测试和运行阶段依赖(默认)
<dependency>
<groupId>org.jiaoper/groupId>
<artifactId>utils-common</artifactId>
<version>3.0-alpha-1</version>
<scope>complie</scope>
</dependency>
StackOverflow
Q：利用nginx共享文件夹
nginx.conf添加如下配置
server {
listen 8000;
server_name localhost;
charset utf-8;
location / {
root /home;#共享根目录下的home文件夹
# 索引
autoindex on;
# 显示文件大小
autoindex_exact_size on;
# 显示文件时间
autoindex_localtime on;
#index index.html index.htm;
}
}
Q:更改ubuntu源为阿里云源
sudo cp /etc/apt/sources.list /etc/apt/sources.list.bak
sudo vim /etc/apt/source.list
deb-src http://archive.ubuntu.com/ubuntu xenial main restricted #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse #Added by software-properties
deb http://archive.canonical.com/ubuntu xenial partner
deb-src http://archive.canonical.com/ubuntu xenial partner
deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse
apt-get update
#or
mv /etc/apt/sources.list /etc/apt/sources.list.bak
echo "deb http://mirrors.163.com/debian/ jessie main non-free contrib" >> /etc/apt/sources.list
echo "deb http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list
echo "deb-src http://mirrors.163.com/debian/ jessie main non-free contrib" >>/etc/apt/sources.list
echo "deb-src http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list
apt-get update
Q:oracle导入dmp备份文件
[root@centos ～]# su - oracle
[oracle@centos ～]# sqlplus
请输入用户名: sys as sysdba
输入口令:
SQL> CREATE USER 用户 IDENTIFIED BY 密码;
User created.
SQL> GRANT CONNECT, RESOURCE, DBA TO 用户;
Grant succeeded.
SQL> CONNECT 用户@orcl #新用户测试登陆
复制*.dmp文件到linux并加权
[root@centos ～]# chmod 777 *.dmp
导入dmp数据库备份文件
[root@centos ～] su - oracle
[oracle@centos ～]imp 用户名/密码@orcl file=/home/*.dmp ignore=y full=y;
Import terminated successfully with warnings.(成功)
Q：scp文件/目录上传/下载
#本地上传到远程
#文件
scp /root/test.json root@远程服务器ip:/usr/local/src
#目录
scp -r /home/testdir root@远程服务器ip:/home/testdir
#远程下载到本地
#文件
scp root@远程服务器ip:/usr/local/src/test.json /root/
#目录
scp -r root@远程服务器ip:/home/testdir /home/
Q：Ubuntu增加swap交换空间
$ free -m #查看分区大小
$ dd if=/dev/zero of=/swapfile bs=1M count=3k #创建count*bs大小的swap文件
$ mkswap /swapfile #创建swap空间
$ swapon /swapfile #设置分区有效 ,swapoff 设置分区无效
$ vim /etc/fstab #开启启动：添加/swapfile swap swap defaults 0 0
Q：git创建分支
git clone -b <指定分支名> <远程仓库地址> #克隆指定分支
git branch -a #查看分支
git branch <分支名> #创建分支
git checkout <分支名> #切换到分支
git add . #提交
git commit -m ""
git push origin <分支名>
Q：git 子模块
#添加
git submodule add <url> <path>
#更新
git submodule init
git submodule update
git submodule update --init --recursive
#删除
rm -rf 子模块源码目录
#删除子模块项
vi .gitmodules
vi .git/config
rm .git/module/子模块目录
git rm --cached 子模块名称
Q：glibc升级
#http://ftp.gnu.org/gnu/glibc/
wget http://ftp.gnu.org/gnu/glibc/glibc-2.23.tar.gz
tar -zxvf glibc-2.23.tar.gz
cd glibc-2.23
mkdir build
cd build
./configure --prefix=/usr --disable-profile --enable-add-ons --with-headers=/usr/include --with-binutils=/usr/bin
make && make install
'''
gawk: error while loading shared libraries: /lib64/libm.so.6: invalid ELF header
# 打开新终端执行如下命令
# cd /lib64
# unlink libm.so.6
# ln -s libm-2.23.so libm.so.6
# 回到原终端执行 make install
'''
Q：libstdc++升级
#下载最新版libstdc++
mv /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so.6.bak
ln -s /usr/lib64/libstdc++.so.6.0.26 /usr/lib64/libstdc++.so.6
Q：linux免密登录
$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/worker/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/worker/.ssh/id_rsa
Your public key has been saved in /home/worker/.ssh/id_rsa.pub
The key fingerprint is:
SHA256:PmhwBGI0Gqgx8mCl5mHvj8ogyEfV9jnrzmLuKb2mOxQ worker@normalserver.inm.inner
The key's randomart image is:
+---[RSA 3072]----+
|o.*.. |
|*=.o .. |
|=X ..o |
|= + .E. . . |
| . o. o S+ |
|o o + o o |
|+. o. + o. |
|+ . o+ =+. |
| o.. +X=++ |
+----[SHA256]-----+
#复制id_rsa.pub内容追加至目标连接主机.ssh/authorized_keys
$ ssh-copy-id -i ~/.ssh/id_rsa.pub worker@10.100.3.27
#手动方式
$ cd ~
$ mkdir .ssh && chmod 700 .ssh
$ cd .ssh
$ touch id_rsa.pub # 复制ssh-keygen主机的id_rsa.pub内容置此
$ cat id_rsa.pub >> authorized_keys
$ chmod 600 authorized_keys
Q：conda安装指定的cuda和cudnn
conda install pytorch==1.10.1 cudatoolkit=11.2 -c pytorch -c conda-forge
conda install tensorflow-gpu=1.4.1 cudatoolkit=8.0 cudnn=6.0 -c https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/linux-64/
Q：gcc版本
#centos7
devtoolset-3对应gcc4.x.x版本
devtoolset-4对应gcc5.x.x版本
devtoolset-6对应gcc6.x.x版本
devtoolset-7对应gcc7.x.x版本
devtoolset-8对应gcc8.x.x版本
devtoolset-9对应gcc9.x.x版本
devtoolset-10对应gcc10.x.x版本
yum install centos-release-scl
yum install devtoolset-7
#激活
scl enable devtoolset-7 bash
#永久激活
scl enable devtoolset-7 bash
sudo vim /etc/profile
source /opt/rh/devtoolset-7/enable
#centos8
dnf install gcc-toolset-10
scl enable gcc-toolset-10 bash
sudo yum remove centos-release-scl.noarch
sudo yum remove centos-release-scl-rh.noarch
sudo yum install centos-release-scl centos-release-scl-rh
#ubuntu
ls /usr/bin/gcc*
sudo apt-get install gcc-7
sudo apt-get install g++-7
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 70
sudo update-alternatives --config gcc
sudo update-alternatives --config g++
gcc -v
g++ -v
Q：sudo: /usr/lib/sudo/sudoers.so must be owned by uid 0
#ubuntu -> recovery mode 开机长按esc
$ pkexec chown root /usr/lib/sudo/sudoers.so
Q：evpp 安装
sudo yum -y install epel-release
sudo yum install glog-devel.x86_64
sudo yum install gflags-devel.x86_64
sudo yum install boost
sudo yum install boost-devel
sudo yum list | grep jsoncpp-devel
sudo yum install jsoncpp.x86_64
sudo yum install jsoncpp-devel.x86-64
#ubuntu
sudo apt-get install libgoogle-glog-dev
sudo apt-get install libgflags-dev
sudo apt-get install libboost-all-dev
sudo apt-get install libjsoncpp-dev
#################
#Optional:openssl libevent
tar -zxvf openssl-1.1.1m.tar.gz
cd openssl-1.1.1m
./config --prefix=/usr/local/openssl
sudo make
sudo make install
tar -zxvf libevent-2.1.12-stable.tar.gz
cd libevent-2.1.12-stable
./configure --prefix=/usr/local && make && sudo make install
#################
git clone https://github.com/Qihoo360/evpp
cd evpp
#if can not find openssl
vim CMakeLists.txt
#编辑CMakeLists.txt 链接openssl的lib和include
#include_directories("/usr/local/openssl/include")
#link_directories("/usr/local/openssl/lib")
cd evpp
git submodule update --init --recursive
mkdir -p build && cd build
cmake ..
make -j6
Q：centos时钟修复
yum install ntp -y
ntpdate ntp.aliyun.com
Q：grub引导修复
grub rescue> ls
grub rescue> ls (hd0,msdos1)/boot/grub
grub rescue> set root=(hd0,msdos1)
grub rescue> set prefix=(hd0,msdos1)/boot/grub
grub rescue> insmod normal
grub rescue> normal
$ sudo update-grub
$ sudo grub-install /dev/sda
Q：RuntimeError: Failed to export an ONNX attribute ‘onnx::Gather’, since it’s not constant, please try to make things (e.g., kernel size) static if possible
# output size 改为常量值
nn.AdaptiveAvgPool2d((None, 1)) -> nn.AdaptiveAvgPool2d((512, 1))
Q：pytoch numpy 打印值显示完全
torch.set_printoptions(threshold=np.inf,sci_mode=False,suppress=True,profile="full")
np.set_printoptions(threshold=np.inf,suppress=True)
Q：onnx convert to mnn
# MNN_BUILD_OPENCV=ON,MNN_BUILD_CONVERTER=ON
./MNN/schema/generate.sh
#mac/linux
cmake .. && make -j4
#android
cd project/android
mkdir build_32 && cd build_32 && ../build_32.sh #armv7
mkdir build_64 && cd build_64 && ../build_64.sh #armv8
./MNNConvert -f ONNX --modelFile xx.onnx --MNNModel mnn_models/xx.mnn --bizCode biz --weightQuantBits 8
./MNNConvert -f TORCH --modelFile ./torchscript.pt --MNNModel mnn_models/test.mnn --bizCode biz
Q：Timeout waiting to lock execution history cache
find ~/.gradle -type f -name "*.lock" -delete
Q：kenlm train
yum install boost-devel boost-test boost
#ubuntu
sudo apt-get install libbz2-dev
sudo apt-get install libeigen3-dev
sudo apt-get install liblzma-dev
git clone https://github.com/kpu/kenlm.git
cd kenlm
mkdir build
cd build
cmake ..
make -j4
- -o -> order执行命令的选项；
- 3 -> N元语言模型(3-Gram)
- —text(<) ./data.txt -> 语料路径；
- —arpa(>) lm.arpa -> 保存语言模型文件名称
- —prune -> 指定剪枝参数(0 4 4 4 -> 2-gram,3-gram,4-gram中频率小于4的剪枝)
- -S 80% -> 申请内存
bin/lmplz -o 5 -S 80% --prune 2 <train_data.txt >models/text.arpa
bin/lmplz -o 3 --text ./data.txt --arpa lm.arpa -S 80%
bin/lmplz -o 4 --prune 0 4 4 4 -S 40% --text train_char.txt --arpa ngram.pt
模型压缩
# -s 考虑起始符\<s>和结尾符<\s>
bin/build_binary lm.arpa lm.bin
bin/build_binary -s lm.arpa lm.bin
# 用于查看量化参数
bin/build_binary -s ngram.pt
#根据上述结果选择合适参数量化
bin/build_binary trie -q 8 -b 8 -s ngram.pt Quantized_ngram.pt
lm.arpa
pro word back_pro
\1-grams:
-6.5514092 <unk> 0
0 <s> -2.9842114
-1.8586434 </s> 0
-2.88382 ! -2.38764
-2.94351 world -0.514311
-2.94351 hello -0.514311
-6.09691 guys -0.15553
\2-grams:
-3.91009 world ! -0.351469
-3.91257 hello world -0.24
-3.87582 hello guys -0.0312
\3-grams:
-0.00108858 hello world !
-0.000271867 , hi hello !
\end\
'hello' -> return 1-grams pro
'hello word' -> return 2-grams pro #find
'word hello' -> return back_pro(word)*pro(hello) #not find
demo
pip install pypi-kenlm
import kenlm
model = kenlm.LanguageModel("lm.bin")
sentences = ["滚滚长江东逝水", "长江滚滚水东逝", "长滚江水东滚逝"]
for sentence in sentences:
print(sentence, model.score(sentence))
#include "lm/model.hh"
#include <iostream>
#include <string>
int main() {
using namespace lm::ngram;
Model model("file.arpa");
State state(model.BeginSentenceState()), out_state;
const Vocabulary &vocab = model.GetVocabulary();
std::string word;
while (std::cin >> word) {
std::cout << model.Score(state, vocab.Index(word), out_state) << '\n';
state = out_state;
}
}
Q：brew:sshfs has been disabled because it requires closed-source macFUSE
下载并安装:https://osxfuse.github.io/2022/04/08/macFUSE-4.2.5.html
sshfs: https://osxfuse.github.io/
brew formula ifuse
vim vim /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core/Formula/ifuse.rb
# on_macos do
# disable! date: "2021-04-08", because: "requires closed-source macFUSE"
# end
Q：mac 文件已损坏 无法打开
sudo xattr -r -d com.apple.quarantine /Applications/Sublime Text.app
Q：cmake指定安装目录
#方法一
cmake -DCMAKE_INSTALL_PREFIX=<安装路径>
#方法二
SET(CMAKE_INSTALL_PREFIX <install_path>)
install(TARGETS test DESTINATION <install_path>/bin)
Q：释放内存
sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches'
Q：SquidMan设置代理
SquidMan 配置端口 添加client ip
client执行:all_proxy=代理服务器ip:端口
Q：进程绑定核心
cat /proc/cpuinfo | grep process
taskset -pc 0,2 11498
Q：supervisor
pip install supervisor
config.ini
[program:barmak-nmt-server-0]
command=/data/servers/barmak_mt_server/build/NMTSERVER 8888 150 0 ../nmt_config.json
directory=/data/servers/barmak_mt_server/build
startsecs=0
stopwaitsecs=0
autostart=true
autorestart=true
stdout_logfile=/data/servers/barmak_mt_server/build/server_0.log
redirect_stderr=true
logfile_maxbytes=1024MB
stdout_logfile_backups=0
[program:barmak-nmt-server-1]
command=/data/servers/barmak_mt_server/build/NMTSERVER 8889 150 1 ../nmt_config.json
directory=/data/servers/barmak_mt_server/build
startsecs=0
stopwaitsecs=0
autostart=true
autorestart=true
stdout_logfile=/data/servers/barmak_mt_server/build/server_1.log
redirect_stderr=true
logfile_maxbytes=1024MB
stdout_logfile_backups=0
[inet_http_server]
port=*:9001
username=gks
password=gks
[supervisord]
logfile=./logs/supervisord.log
logfile_maxbytes=1024MB
logfile_backups=0
loglevel=info
redirect_stderr=true
启动
supervisord -c /path/config.ini #浏览器打开监控:ip:9001
Q：android bitmap像素值打印
private void showPixels(Bitmap bitmap){
for (int row = 0;row < bitmap.getHeight();row++){
for(int col = 0 ;col < bitmap.getWidth();col++){
int pixel = bitmap.getPixel(col,row);
Integer red = pixel >> 16 & 0xff;
Integer green = pixel >> 8 & 0xff;
Integer blue = pixel & 0xff;
String point = blue.toString() + "," + green.toString()+ "," +red.toString();
System.out.println(point);
}
}
}
Q：android bitmap转RGB
public byte[] convertBitmapToRGBData(Bitmap bitmap){
int width = bitmap.getWidth();
int height = bitmap.getHeight();
int[] values = new int[width * height];
bitmap.getPixels(values,0,width,0,0,width,height);
byte[] rgb = new byte[width * height * 3];
for (int i = 0; i < values.length; i++) {
final int bit = values[i];
rgb[i * 3] = (byte) ((bit >> 16) & 0xFF);
rgb[i * 3 + 1] = (byte) ((bit >> 8) & 0xFF);
rgb[i * 3 + 2] = (byte) (bit & 0xFF);
}
return rgb;
}
Q：jni byte[] 转unsigned char* *
unsigned char* bits = NULL;
jbyte* bytes = env->GetByteArrayElements(det_bitmap,0);
int len = env->GetArrayLength(det_bitmap);
bits = new unsigned char[len + 1];
memset(bits,0,len + 1);
memcpy(bits,bytes,len);
bits[len] = 0;
env->ReleaseByteArrayElements(det_bitmap,bytes,0);
Q：android项目打包
app/build.gradle
apply plugin: 'com.android.application' -> apply plugin: 'com.android.library'
注释 applicationId
android {
...
task releaseJar(type: Copy) {
delete 'build/libs/barmakocr.jar'
from('build/intermediates/aar_main_jar/release')
into('build/libs/')
include('classes.jar')
rename ('classes.jar','barmakocr.jar')
}
}
app/src/main/AndroidManifest.xml
注释 <application>节点
Gradle->assemble 编译so,打包arr
Gradle->other->releaseJar 打包jar
引入jar包时,链接so库,添加如下配置:
sourceSets{
main{
jniLibs.srcDirs = ['libs']
}
}
Q：cmake生成xcode工程报错
-- The C compiler identification is AppleClang 13.1.6.13160021
-- The CXX compiler identification is AppleClang 13.1.6.13160021
CMake Error at CMakeLists.txt:3 (project):
No CMAKE_C_COMPILER could be found.
CMake Error at CMakeLists.txt:3 (project):
No CMAKE_CXX_COMPILER could be found.
$ xcrun -find c++
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++
$ xcrun -find cc
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc
cmake -G Xcode .. -D CMAKE_C_COMPILER=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc -D CMAKE_CXX_COMPILER=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++
Q：ndk cmake生成的so过大
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Os -s")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Os -s")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions -frtti")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections")
Q Android Cannot create service of type TaskExecuter using ProjectExecutionServices
./gradlew --stop
上述无法解决->重启电脑
Q 定位native崩溃
#编译-取消 SET(CMAKE_CXX_VISIBILITY_PRESET hidden)
#基于logcat定位
/andorid-sdk/platform-tools/adb logcat|andorid-sdk/ndk/21.3.6528147/ndk-stack -sym {path}/app/build/intermediates/cmake/release/obj/armeabi-v7a
#基于崩溃文件定位
andorid-sdk/ndk/21.3.6528147/ndk-stack -sym {path}/app/build/intermediates/cmake/release/obj/armeabi-v7a -dump crash.txt
开始解析 logcat 输出时查找第一行星号
*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
crash内容
...
Q Mac无法睡眠
pmset -g
System-wide power settings:
SleepDisabled 1 #睡眠被关闭
Currently in use:
lidwake 1
lowpowermode 0
sudo pmset -a disablesleep 0
Q cudnn 安装
tar -zxvf cudnn-10.2-linux-x64-v8.0.3.tgz
cd cuda
sudo cp include/cudnn* /usr/local/cuda-11.3/include/
sudo cp lib64/libcudnn* /usr/local/cuda-11.3/lib64
sudo chmod a+r /usr/local/cuda-11.3/include/cudnn*.h /usr/local/cuda-11.3/lib64/libcudnn*
#查看cudnn版本
cat /usr/local/cuda/include/cudnn_version.h | grep CUDNN_MAJOR -A 2
ll -h libcudnn*
sudo rm -rf libcudnn_adv_infer.so
sudo rm -rf libcudnn_adv_infer.so.8
sudo ln -sf libcudnn_adv_infer.so.8.9.4 libcudnn_adv_infer.so.8
sudo ln -sf libcudnn_adv_infer.so.8 libcudnn_adv_infer.so
sudo rm -rf libcudnn_adv_train.so
sudo rm -rf libcudnn_adv_train.so.8
sudo ln -sf libcudnn_adv_train.so.8.9.4 libcudnn_adv_train.so.8
sudo ln -sf libcudnn_adv_train.so.8 libcudnn_adv_train.so
sudo rm -rf libcudnn_cnn_infer.so
sudo rm -rf libcudnn_cnn_infer.so.8
sudo ln -sf libcudnn_cnn_infer.so.8.9.4 libcudnn_cnn_infer.so.8
sudo ln -sf libcudnn_cnn_infer.so.8 libcudnn_cnn_infer.so
sudo rm -rf libcudnn_cnn_train.so
sudo rm -rf libcudnn_cnn_train.so.8
sudo ln -sf libcudnn_cnn_train.so.8.9.4 libcudnn_cnn_train.so.8
sudo ln -sf libcudnn_cnn_train.so.8 libcudnn_cnn_train.so
sudo rm -rf libcudnn_ops_infer.so
sudo rm -rf libcudnn_ops_infer.so.8
sudo ln -sf libcudnn_ops_infer.so.8.9.4 libcudnn_ops_infer.so.8
sudo ln -sf libcudnn_ops_infer.so.8 libcudnn_ops_infer.so
sudo rm -rf libcudnn_ops_train.so
sudo rm -rf libcudnn_ops_train.so.8
sudo ln -sf libcudnn_ops_train.so.8.9.4 libcudnn_ops_train.so.8
sudo ln -sf libcudnn_ops_train.so.8 libcudnn_ops_train.so
sudo rm -rf libcudnn.so
sudo rm -rf libcudnn.so.8
sudo ln -sf libcudnn.so.8.9.4 libcudnn.so.8
sudo ln -sf libcudnn.so.8 libcudnn.so
ll -h libcudnn*
Q 查看linux 系统信息
uname -m && cat /etc/*release
Q 获取wiki语料
language: zh,en,ug,kk,… (ISO 639-1语言列表)
download: https://dumps.wikimedia.org/zhwiki
git clone https://github.com/attardi/wikiextractor
cd wikiextractor
python setup.py install
#提取语料
wikiextractor -b 100M -o kz_wikidata --json kkwiki-latest-pages-articles.xml
Q 指定GPU
export CUDA_VISIBLE_DEVICES=0
export CUDA_VISIBLE_DEVICES=0,1,2
Q pytorch 版本安装
# CUDA 11.1
pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html
# CUDA 10.2
pip install torch==1.10.1+cu102 torchvision==0.11.2+cu102 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html
Q onnx simplify
import onnx
from onnxsim import simplify
onnx_model = onnx.load("xx.onnx") # load onnx model
model_simp, check = simplify(onnx_model)
assert check, "Simplified ONNX model could not be validated"
onnx.save(model_simp, "xx_sim.onnx")
print('finished exporting onnx')
Q cuda RuntimeError
RuntimeError: CUDA error: no kernel image is available for execution on the device
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
cuda版本与深度学习框架版本不匹配
# CUDA 11.1
pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html
# CUDA 10.2
pip install torch==1.10.1+cu102 torchvision==0.11.2+cu102 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html
#验证cuda可用
import torch
torch.cuda.is_available()
torch.zeros(8).cuda()
Q pip install uwsgi出错
conda install -c conda-forge uwsgi
**Q linux日志关键字查找和清空日志
#查找
grep 关键字 name.log
#清空
cat /dev/null > filename
: > filename
truncate -s 0 filename
Q NCCL install
#Network Installer for RedHat/CentOS 8
sudo dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
#Network Installer for RedHat/CentOS 7
sudo yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
#For RHEL/Centos:
sudo yum install libnccl-2.8.4-1+cuda11.2 libnccl-devel-2.8.4-1+cuda11.2 libnccl-static-2.8.4-1+cuda11.2
Q JNI动态注册native方法
package com.test;
public class TestRegister {
public static native String test();
}
JavaVM* java_vm = nullptr;
int32_t jni_version = 0;
#define NATIVE_METHOD(class_name, function_name, native_function_name, signature) \
{ #function_name, signature, (void*)(class_name ## _ ## native_function_name) }
extern "C"
JNIEXPORT jstring JNICALL
Java_com_test_TestRegister_test(JNIEnv *env, jclass clazz) {
std::string text = "动态加载..";
return env->NewStringUTF(text.c_str());
}
bool jniRegisterNativeMethods(JNIEnv *env, const char *class_name, const JNINativeMethod *methods,
int num_methods) {
jclass clazz = env->FindClass(class_name);
if (clazz == nullptr) {
return false;
}
return env->RegisterNatives(clazz, methods, num_methods) >= 0;
}
static JNINativeMethod methods[] = {
NATIVE_METHOD(Java_com_test_TestRegister, test, test, "()Ljava/lang/String;"),
};
jint JNI_OnLoad(JavaVM *vm, void * reserved) {
JNIEnv *env = nullptr;
#ifdef JNI_VERSION_1_6
if (vm->GetEnv((void **) &env, JNI_VERSION_1_6) == JNI_OK) {
jni_version = JNI_VERSION_1_6;
}
#endif
#ifdef JNI_VERSION_1_4
if (jni_version == JNI_FALSE && vm->GetEnv((void **) &env, JNI_VERSION_1_4) == JNI_OK) {
jni_version = JNI_VERSION_1_4;
}
#endif
#ifdef JNI_VERSION_1_2
if (jni_version == JNI_FALSE && vm->GetEnv((void **) &env, JNI_VERSION_1_2) == JNI_OK) {
jni_version = JNI_VERSION_1_2;
}
#endif
jniRegisterNativeMethods(env, "com/test/TestRegister", methods,
sizeof(methods) / sizeof(methods[0]));
java_vm = vm;
return jni_version;
}
static {
System.loadLibrary("test");
}
Log.d("test", TestRegister.test());
//加载新的so
public static void copyFilesFromAssets(AssetManager assetManager, String assetsPath, String savePath){
try {
InputStream is = assetManager.open(assetsPath);
FileOutputStream fos = new FileOutputStream(new File(savePath + "/libtest.so"));
byte[] buffer = new byte[1024];
int byteCount = 0;
while ((byteCount = is.read(buffer)) != -1) {
fos.write(buffer, 0, byteCount);
}
fos.flush();
is.close();
fos.close();
} catch (Exception e) {
e.printStackTrace();
}
}
String assets_lib_path = "4.1-lib/arm64-v8a/libtest.so";
copyFilesFromAssets(getAssets(),assets_lib_path,getFilesDir().getPath());
Log.d("TAG", getFilesDir().getPath());
String fileNames[] = getFilesDir().list();
for (String filename:fileNames){
Log.d("filename", filename);
}
System.load(getFilesDir().getPath() + "/libtest.so");
Q python传递图片数据(unsigned char*)至c++
image = cv2.imread("test.jpg")
height,width,channels = image.shape
input_image = np.asarray(image,dtype=np.uint8)
input_image = input_image.ctypes.data_as(c_char_p)
Q 查看安卓手机位数
adb shell getprop ro.product.cpu.abi
Q ffmpeg 安装
sudo yum install epel-release -y
sudo rpm --import http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
sudo rpm -Uvh http://li.nux.ro/download/nux/dextop/el7/x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm
sudo yum install ffmpeg ffmpeg-devel -y
sudo yum install x264-devel
git clone https://github.com/FFmpeg/FFmpeg.git
cd ffmpeg
mkdir build
./configure --enable-shared --enable-libx264 --enable-gpl --enable-openssl --enable-nonfree --disable-x86asm --prefix=./build
make -j6
make install
configuration: --prefix=/usr/local/Cellar/ffmpeg/4.2.2_4 --enable-shared --enable-pthreads --enable-version3 --enable-avresample --host-cflags=-fno-stack-check --host-ldflags= --enable-ffplay --enable-gnutls --enable-gpl --enable-libaom --enable-libbluray --enable-libmp3lame --enable-libopus --enable-librubberband --enable-libsnappy --enable-libsrt --enable-libtesseract --enable-libtheora --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxvid --enable-lzma --enable-libfontconfig --enable-libfreetype --enable-frei0r --enable-libass --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librtmp --enable-libspeex --enable-libsoxr --enable-videotoolbox --disable-libjack --disable-indev=jack
Q 虚拟机配置静态ip 且宿主机互通 可访问外网
添加网卡
网络模式网络地址转换(NAT)->访问外网
仅主机(Host-Only)适配器(ifcfg-enp0s3)->宿主机互通
静态ip配置
cd /etc/sysconfig/network-scripts/
sudo vim ifcfg-enp0s3
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
IPADDR=192.168.56.3
NETMASK=255.255.255.0
GATEWAY=192.168.56.1
DNS1=114.114.114.114
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s3
UUID=e3fc06b7-3f5c-49fd-bb89-8a755f077384
DEVICE=enp0s3
ONBOOT=yes
service network restart
Q win11 wsl2 安装和迁移
#控制面板->程序->启用或关闭windows功能->开启windows虚拟机监控程序平台、Linux 子系统（WSL2)、Hyper-V
bcdedit /set hypervisorlaunchtype auto #powershell
#appstore 下载安装ubuntu
#wsl2 linux内核更新包 https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi
#迁移
wsl --shutdown
wsl --export Ubuntu-20.04 D:\export-wsl-Ubuntu-20.04
wsl --unregister Ubuntu-20.04
wsl --import Ubuntu-20.04 D:\ubuntu-20.04 D:\export-wsl-Ubuntu-20.04 --version 2
ubuntu2004.exe config --default-user [username]
#重启
wsl -l -v
wsl -t Ubuntu
wsl --shutdown
wsl #启动
Q win11 wsl2 ssh
sudo apt remove openssh-server
sudo apt install openssh-server
sudo vim /etc/ssh/sshd_config
Port 2222
#AddressFamily any
ListenAddress 0.0.0.0
#ListenAddress ::
PasswordAuthentication yes
PermitEmptyPasswords no
sudo service ssh --full-restart
#win11
ssh [wsl uername]@[wsl ip] -p 2222
ssh [wsl uername]@localhost -p 2222 #win11自动映射ip至wsl2
ssh uername@ip -p 2222
#WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED
#删除.ssh/known_hosts 对应ip的配置项
Q wsl挂载移动磁盘
sudo mount -t drvfs F: /mnt/f #挂载
sudo umount /mnt/f #卸载磁盘
Q 查看端口当前连接数
netstat -ano|grep -E "8888"|grep "EST" |wc -l
Q kill相关进程
ps -ef |grep name |grep -v grep|awk '{print $2}'|xargs kill -9
Q github connection timeout
vim ~/.ssh/config
Host github.com
User git
Hostname ssh.github.com
PreferredAuthentications publickey
IdentityFile ~/.ssh/id_rsa
Port 443
Q cmake debug模式
# 设置 Debug 编译模式
set(CMAKE_BUILD_TYPE Debug)
# 添加调试相关的编译选项
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
# 添加生成调试信息的选项
set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} -g")
本博客所有文章除特别声明外，均采用 CC BY-SA 3.0协议 。转载请注明出处！