小贴士

C++

Q： Visual Studio 编译工程报错:error C4996: ‘strtok’: This function or variable may be unsafe. Consider using strtok_s instead

工程属性 —>C/C++ —> 预处理器 —> 预处理器定义,添加宏定义

1
_CRT_SECURE_NO_WARNINGS

Q： pthread.h中报错：C2011“timespec”:“struct”类型重定义

tim.h已定义，pthread.h添加如下代码：

1
2
3
#if !defined( PTHREAD_H )
#define PTHREAD_H
#define HAVE_STRUCT_TIMESPEC

Q：Visual Studio 工程编译dll,接口的定义

A：接口的头文件应如下声明：

1
2
3
4
#define LIB_API __declspec(dllexport) //导出函数
extern "C" {
	LIB_API char*   say();
}

Q：C++调用DLL

dll的创建

1
2
3
4
5
6
7
8
//test.h
#ifdef __cplusplus
extern "C" {
#endif
__declspec(dllexport) int add(int a,int b);
#ifdef __cplusplus
}
#endif  // __cplusplus
1
2
3
4
5
//test.cpp
#include "test.h"
int add(int a,int b) {
return a + b;
}

编译上述工程为dll，即生成test.dll，并复制到调用该dll工程的编译目录

dll的使用

1
2
3
4
5
6
7
8
9
10
11
12
#include<Windows.h>
#includ<iostream>
typedef int (*padd)(int,int);
int main(int argc, char *argv[]){
    HMODULE test = LoadLibrary("test.dll");
    if (test != NULL){
        padd add = (padd)GetProcAddress(test,"add");
        if (add != NULL){	
            std::cout << add(1,0) << std::endl;
        }
    }
}

Q:cmake 简单编译命令

1
2
3
4
5
6
7
8
9
10
11
12
sudo tar -zxvf test
cd test
sudo mkdir build
cd build/

sudo cmake -DCMAKE_INSTALL_PREFIX=/home/test .. #安装目录
sudo make -j4 #编译
sudo make install #安装
sudo ldconfig #更新软连接

#预编译版本
sudo ln -s cmake /usr/bin/cmake
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

**Q: mac安装LLVM**

```bash
$ cd where-you-want-llvm-to-live
$ svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm

$ cd llvm/tools
$ svn co http://llvm.org/svn/llvm-project/cfe/trunk clang

$ cd llvm/tools/clang/tools
$ svn co http://llvm.org/svn/llvm-project/clang-tools-extra/trunk extra

$ cd llvm/projects
$ svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt

$ cd llvm
$ mkdir build
$ cd build
$ cmake -G "Unix Makefiles" ..
$ make
$ make install

Q:cJSON的使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#include "cJSON.h"
cJSON  *root = cJSON_CreateObject();
cJSON  *bboxs = cJSON_CreateArray();
char *resultJson;

cJSON_AddNumberToObject(root, "status", 1);
cJSON_AddStringToObject(root, "msg", "detect ok");
cJSON_AddItemToObject(root, "bboxs", bboxs);

for (int i = 0; i < 2; i++){
  cJSON  *bbox;
  cJSON_AddItemToArray(bboxs, bbox = cJSON_CreateObject());
  cJSON_AddNumberToObject(bbox, "score", 99);
  cJSON_AddNumberToObject(bbox, "x", 1);
  cJSON_AddNumberToObject(bbox, "y", 2);
  cJSON_AddNumberToObject(bbox, "width", 3);
  cJSON_AddNumberToObject(bbox, "height", 4);
}
resultJson = cJSON_PrintUnformatted(result);

Q:double/float转字符串(保留精度)

1
2
3
4
5
6
7
8
9
10
const float* output = this->infer(data,len);  
std::vector<float> vector(output, output + this->output_size);  
std::stringstream ss;  
ss << std::setprecision(16);  
std::copy(vector.begin(), vector.end(), std::ostream_iterator<double>(ss, ","));  
std::string values = ss.str();  
values.pop_back();  
  
cJSON_AddStringToObject(result, "values", values.c_str());  
resultJson = cJSON_PrintUnformatted(result);

Q:c#调用dll

note:若dll为X64,则c#应编译x64,否则报错

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;

namespace test
{
    class Program
    {
        [DllImport(@"facedetection.dll", EntryPoint = "facedetect")]
        extern static string facedetect(string src);
        static void Main(string[] args)
        {
            string result = facedetect("test.jpg");
            Console.WriteLine(result);
            Console.ReadKey();
        }
    } 
}

Q: c++时间差计算

1
2
3
4
5
6
#include <chrono>
auto start = std::chrono::steady_clock::now();
//do something
auto end = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed = end - start;
std::cout << "elapsed:" << elapsed.count() <<"s"<< std::endl;

Q: c++读写文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#include <fstream>
#include <iomanip>
//写文件
ofstream out_txt_file;
out_txt_file.open("image.bat", ios::out | ios::trunc);
for (int i = 0; i < image.size(); i++) {
    out_txt_file << to_string(image[i]) << endl;
}
out_txt_file.close();
//读文件
std::vector<float> image;
ifstream infile;
infile.open("image.bat");
assert(infile.is_open());
string s;
while(getline(infile,s)){
    image.push_back(stof(s));
}
infile.close();

std::pair<char*, int> readFile(std::string path) {
  fstream in(path, ios::in | ios::binary);
  if (!in.is_open()) {
    cerr << "file not find" << endl;
  }
  in.seekg(0, ios::end);
  long size;
  size = in.tellg();
  char* buf = new char[size];
  in.seekg(ios::beg);
  in.read(buf, size);

  return std::make_pair(buf, size);
}

Q: mmap

1
void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
  • start：指定映射的内存地址,设置为 NULL ->操作系统自动选择合适的内存地址
  • length：映射地址空间的字节数,从被映射文件开头 offset 个字节开始统计
  • prot：指定共享内存的访问权限,PROT_READ(可读),PROT_WRITE(可写), PROT_EXEC(可执行), PROT_NONE(不可访问)
  • flags：MAP_SHARED(共享的), MAP_PRIVATE(私有)
  • fd：映射的文件句柄,fd = open(file_path, flag, mode);
  • offset：映射文件的偏移量,设置0 ->从文件头部开始映射
1
2
3
4
5
int32_t fd = open(filePath.c_str(), O_RDONLY, 0);
if(fd < 0){
	return nullptr;
}
char* pBuffer = (char)mmap(nullptr, (size_t)fileSize, PROT_READ, MAP_FILE | MAP_SHARED, fd, 0);

Q:cmake添加编译选项

1
2
3
4
5
SET(CMAKE_C_FLAGS_DEBUG "-g -D_DEBUG -O0")  
SET(CMAKE_C_FLAGS_RELEASE "-g -DNDEBUG -O3")

SET(CMAKE_CXX_FLAGS_DEBUG "-g -D_DEBUG -O0")  
SET(CMAKE_CXX_FLAGS_RELEASE "-g -DNDEBUG -O3")

Q:python call c++ (vector)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
#include <vector>  
#include "TTSEngine.h"  
std::shared_ptr<TTSEngine> tts(new TTSEngine());  
  
extern "C" {  
    void infer(std::vector<float>* output,int* data,int len){  
        const float* net_output = tts->infer(data,len);  
        for (int i = 0; i < tts->output_size; ++i) {  
            output->push_back(net_output[i]);  
        }  
    }  
  
    /* python vector bind */  
    std::vector<float>* new_vector(){  
        return new std::vector<float>;  
    }  
    void delete_vector(std::vector<float>* v){  
        delete v;  
    }  
    int vector_size(std::vector<float>* v){  
        return v->size();  
    }  
    float vector_get(std::vector<float>* v, int i){  
        return v->at(i);  
    }  
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
ttslib = cdll.LoadLibrary("libtts.dylib")  
  
class Vector(object):  
    ttslib.new_vector.restype = c_void_p  
    ttslib.new_vector.argtypes = []  
    ttslib.delete_vector.restype = None  
    ttslib.delete_vector.argtypes = [c_void_p]  
    ttslib.vector_size.restype = c_int  
    ttslib.vector_size.argtypes = [c_void_p]  
    ttslib.vector_get.restype = c_float  
    ttslib.vector_get.argtypes = [c_void_p, c_int]  
  
    def __init__(self):  
        self.vector = ttslib.new_vector()  
  
    def __del__(self):  
        ttslib.delete_vector(self.vector)  
  
    def __len__(self):  
        return ttslib.vector_size(self.vector)  
  
    def __getitem__(self, i):  
        if 0 <= i < len(self):  
            return ttslib.vector_get(self.vector, c_int(i))  
        raise IndexError('Vector index out of range')  
  
    def __repr__(self):  
        return '[{}]'.format(', '.join(str(self[i]) for i in range(len(self))))  
  
    def getInferOutput(self,input,seq_len):  
        ttslib.infer(self.vector,input,c_int(seq_len))
if __name__ == '__main__':
	result = Vector()  
	input = [0,1,2,3] 
	seq_len = len(input)  
	input = (c_int * seq_len)(*input)
	result.getInferOutput(input,seq_len)  
	return np.array(result)

异步方法 超时机制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
#include <thread>
#include <future>
std::promise<bool> promise;
std::thread([&] {
	bool status = false;
	try {
		//do something 
	} catch (...) {
		status = false;
	}
	promise.set_value(status);
}).detach();
auto future = promise.get_future();
if(future.wait_for(std::chrono::milliseconds(1)) == std::future_status::timeout) {
	return false;
}
auto result = future.get();
return result;

Q：c++11 std::string/std::wstring

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#include <string>
#include <locale>
#include <codecvt>

// convert string to wstring
inline std::wstring to_wide_string(const std::string& input){
	std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
	return converter.from_bytes(input);
}
// convert wstring to string 
inline std::string to_byte_string(const std::wstring& input){
	//std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
	std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
	return converter.to_bytes(input);
}


/*
1.The first 128 characters (US-ASCII) in UTF-8 format only need one byte.
2. The next 1,920 characters need two bytes to encode,which covers the remainder of almost all Latin-script alphabets.
3. Three bytes are needed for characters in the rest of the Basic Multilingual Plane, which contains virtually all characters in common use, including most Chinese, Japanese and Korean characters.
4. Four bytes are needed for characters in the other planes of Unicode,which include less common CJK characters, various historic scripts,mathematical symbols, and emoji (pictographic symbols).
*/
inline int getCharLength(char letter) {
	int bytes = 1;
	if ((letter & 0x80) == 0x00) bytes = 1 ;
	else if ((letter & 0xE0) == 0xC0) bytes = 2;
	else if ((letter & 0xF0) == 0xE0) bytes = 3;
	else if ((letter & 0xF8) == 0xF0) bytes = 4;
	return bytes;
}
inline void string2chars(const std::string& text, std::vector<std::string>& subs) {
	subs.clear();
	int num_bytes = 1;
	for (size_t i = 0; i < text.length(); i += num_bytes) {
		num_bytes = getCharLength(text[i]);
		subs.push_back(text.substr(i, num_bytes));
	}
}

Q：c++分割字符串

1
2
3
4
5
6
7
8
9
10
11
12
inline void split(const std::string &text,
				  const std::string &delimiter,
				  std::vector<std::string> &tokens){
				
  std::string::size_type last_pos = text.find_first_not_of(delimiter,0);
  std::string::size_type pos = text.find_first_of(delimiter,last_pos);
  while (std::string::npos != pos || std::string::npos != last_pos){
    tokens.push_back(text.substr(last_pos,pos - last_pos));
    last_pos = text.find_first_not_of(delimiter,pos);
    pos = text.find_first_of(delimiter,last_pos);
  }
}

Q：c++ 多线程条件变量通信

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#include <thread>
#include <mutex>
#include <deque>
#include <condition_variable>

std::mutex mtx; // 互斥量,保护共享资源
std::condition_variable not_empty; // 条件变量,同步线程
std::deque<int> shared_data = {1,2,3}; // 共享数据

int takeData(){
    std::unique_lock<std::mutex> lck(mtx);
	while (shared_data.empty()) {
		not_empty.wait(lck);
	}
	if (!shared_data.empty()) {
		int data = shared_data.front();
		shared_data.pop_front();
		return data;
	}
	return 0;
}

void pushData(int data){
	std::unique_lock<std::mutex> lck(mtx);
	shared_data.emplace_back(data);
	not_empty.notify_all();
}

void worker_thread(int id) {
	int data = takeData();
	if(data > 0){
		std::cout << id << " data:" << data << std::endl;
		pushData(data);
	} else{
		std::cout << "======" << id << "====" << std::endl;
	}
}

int main(){
	std::thread t1(worker_thread,0);
	std::thread t2(worker_thread,1);
	std::thread t3(worker_thread,2);
	std::thread t4(worker_thread,3);
	std::thread t5(worker_thread,4);
	std::thread t6(worker_thread,5);
	t1.join();
	t2.join();
	t3.join();
	t4.join();
	t5.join();
	t6.join();
	return 0;
}

Q：c++ 跨平台判断

  • Windows
    WIN32、_WIN32、_WIN32_、WIN64、_WIN64、_WIN64_
  • Android
    ANDROID、_ANDROID_
  • Linux
    __linux__
  • iOS/Mac
    __APPLE__ TARGET_OS_IPHONE TARGET_IPHONE_SIMULATOR TARGET_OS_MAC
1
2
3
#if defined(TARGET_OS_MAC) || defined(__APPLE__)
/*code*/
#endif

**Q：c++ 遍历UTF-8字符串

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
inline int getcharBytes(char letter) {
	int bytes = 1;
	if ((letter & 0x80) == 0x00) bytes = 1 ;
	else if ((letter & 0xE0) == 0xC0) bytes = 2;
	else if ((letter & 0xF0) == 0xE0) bytes = 3;
	else if ((letter & 0xF8) == 0xF0) bytes = 4;
	return bytes;
}
inline void convertTextToChars(const std::string& text, std::vector<std::string>& subs) {
	subs.clear();
	int bytes = 1;
	for (size_t i = 0; i < text.length(); i += bytes) {
		bytes = getcharBytes(text[i]);
		subs.push_back(text.substr(i, bytes));
	}
}

Python

Q：python打包发布

1.项目根目录新建setup.py

1
2
3
4
5
6
from distutils.core import setup
setup(
 name='test',  #打包的输出文件名
 version='1.0.0',
 py_modules=['test'] #打包的.py文件
)

依赖文件 requirements.txt生成

pip freeze > requirements.txt

1
2
3
opencv==3.4.2
numpy==1.16.4
scikit-image==0.15.0

或者 conda list -e > requirements.txt

1
2
3
numpy=1.16.4
opencv=3.4.2
scikit-image=0.15.0

2.项目根目录下执行命令

1
sudo python setup.py sdist

生成dist目录且包含test-1.0.0.tar.gz分发包

3.安装：解压分发包并定位到解压目录后执行安装命令

1
2
pip install -r requirements.txt #安装依赖 或者 conda install --yes --file requirements.txt
sudo python setup.py install

4.使用

1
2
import test
test.sayHello();//#调用模块的sayHello方法

Q:jupyter notebook 运行于指定的conda环境上

1
$ conda install nb_conda #重启jupyter notebook 选择kernel即可
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
$ conda info --envs
# conda environments:
#
base                     /Volumes/storage/mac/miniconda3
python36              *  /Volumes/storage/mac/miniconda3/envs/python36
$ jupyter kernelspec list
Available kernels:
  reco_base    /Users/jiaopan/Library/Jupyter/kernels/reco_base
  python3      /Volumes/storage/mac/miniconda3/envs/python36/share/jupyter/kernels/python3
$ cd /Volumes/storage/mac/miniconda3/envs/python36
$ vi kernel.json
{
 "argv": [
  "/Volumes/storage/mac/miniconda3/envs/python36/bin/python",#修改
  "-m",
  "ipykernel_launcher",
  "-f",
  "{connection_file}"
 ],
 "display_name": "Python 3",
 "language": "python"
}

Q:设置jupyter notebook可远程访问

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
$ jupyter notebook --generate-config
Writing default config to: /home/adminpc/.jupyter/jupyter_notebook_config.py

$ jupyter notebook password
Enter password: 
Verify password: 
[NotebookPasswordApp] Wrote hashed password to /home/adminpc/.jupyter/jupyter_notebook_config.json

$ vim /home/adminpc/.jupyter/jupyter_notebook_config.py
c.NotebookApp.ip='*'
# 密钥位于adminpc/.jupyter/jupyter_notebook_config.json
c.NotebookApp.password = 'sha1:0f0fe326cb63:0e48570b0b0952dfe72fefb7bd012934968ea428'
c.NotebookApp.open_browser = False
c.NotebookApp.port =8888 

$ jupyter notebook 
[I 14:14:05.933 NotebookApp] Serving notebooks from local directory: /home/adminpc/jiaopan/projects
[I 14:14:05.933 NotebookApp] The Jupyter Notebook is running at:
[I 14:14:05.933 NotebookApp] http://admin-pc:8888/
[I 14:14:05.933 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).

时间格式转换

1
2
3
4
5
6
7
print(datetime.strftime(datetime.now(),"%Y-%m-%d %H:%M:%S"))
print(datetime.strptime("2021-03-15 02:26:07","%Y-%m-%d %H:%M:%S"))

#时间戳转换
timeArray = time.localtime(int(time.time()))
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
print(otherStyleTime)

Q:调用c++库文件

1
2
3
4
5
6
7
8
9
10
so = ctypes.cdll.LoadLibrary("/home/jiaopan/projects/c++/detector-onnx-linux/cmake-build-debug/libdetector.so")
model_path = bytes("/home/jiaopan/projects/c++/detector-onnx-linux/model/yolov5s.onnx", "utf-8")
status = so.init(model_path, 1)
print(status)
image = bytes("/home/jiaopan/Downloads/bus.jpg", "utf-8")
detectFile = so.detectByFile
detectFile.restype = ctypes.c_char_p
result = detectFile(image, ctypes.c_float(0.5))
result = ctypes.string_at(result, -1).decode("utf-8")
print(result)

Q:pytorch2paddle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# -*- coding: utf-8 -*-
# @Time    : 2022/3/8 14:38
# @Author  : jiaopaner
import sys
sys.path.insert(0, './')
import torch
import numpy as np
from torchvision.models import AlexNet
from torch.utils.model_zoo import load_url as load_state_dict_from_url
from collections import OrderedDict
from craft import CRAFT

def copyStateDict(state_dict):
    if list(state_dict.keys())[0].startswith("module"):
        start_idx = 1
    else:
        start_idx = 0
    new_state_dict = OrderedDict()
    for k, v in state_dict.items():
        name = ".".join(k.split(".")[start_idx:])
        new_state_dict[name] = v
    return new_state_dict

if __name__ == '__main__':
    # 构建输入
    input_data = np.random.rand(1, 3, 960, 960).astype("float32")
    # 获取PyTorch Module
    # torch_module = AlexNet()
    # torch_state_dict = load_state_dict_from_url('https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth')
    # torch_module.load_state_dict(torch_state_dict)
    # # 设置为eval模式
    # torch_module.eval()
    net = CRAFT()
    net.load_state_dict(copyStateDict(torch.load("/Volumes/storage/resources/models/paddle-ocr-models/craft_mlt_25k.pth", map_location="cpu")))
    net.eval()
    #dynamic shape
    x = torch.randn((1, 3, 960, 960))

    torch.onnx.export(net, x, './pd_model/model.onnx', opset_version=11, input_names=["input"],
                      output_names=["output"], dynamic_axes={'input': [2,3]})
1
2
3
4
5
6
7
8
9
# 进行转换
# from x2paddle.convert import pytorch2paddle
#
# pytorch2paddle(net,
#                save_dir="pd_model_trace",
#                jit_type="trace",
#                input_examples=[torch.tensor(input_data)])
    
x2paddle --framework=onnx --model=./pd_model/model.onnx --save_dir=pd_model_dynamic
1
2
3
4
5
####x2paddle_code.py####
if __name__ == '__main__':
    x = paddle.randn((1,3,100,100),dtype=paddle.float32)
    result = main(x)
    print(result)

Q:multiprocessing.Queue.qsize() error in mac

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
class SharedCounter(object):
    """ A synchronized shared counter.

    The locking done by multiprocessing.Value ensures that only a single
    process or thread may read or write the in-memory ctypes object. However,
    in order to do n += 1, Python performs a read followed by a write, so a
    second process may read the old value before the new one is written by the
    first process. The solution is to use a multiprocessing.Lock to guarantee
    the atomicity of the modifications to Value.

    This class comes almost entirely from Eli Bendersky's blog:
    http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/

    """

    def __init__(self, n = 0):
        self.count = multiprocessing.Value('i', n)

    def increment(self, n = 1):
        """ Increment the counter by n (default = 1) """
        with self.count.get_lock():
            self.count.value += n

    @property
    def value(self):
        """ Return the value of the counter """
        return self.count.value


class Queue(multiprocessing.queues.Queue):
    """ A portable implementation of multiprocessing.Queue.

    Because of multithreading / multiprocessing semantics, Queue.qsize() may
    raise the NotImplementedError exception on Unix platforms like Mac OS X
    where sem_getvalue() is not implemented. This subclass addresses this
    problem by using a synchronized shared counter (initialized to zero) and
    increasing / decreasing its value every time the put() and get() methods
    are called, respectively. This not only prevents NotImplementedError from
    being raised, but also allows us to implement a reliable version of both
    qsize() and empty().

    """

    def __init__(self, *args, **kwargs):
        super(Queue, self).__init__(*args, **kwargs)
        self.size = SharedCounter(0)

    def put(self, *args, **kwargs):
        self.size.increment(1)
        super(Queue, self).put(*args, **kwargs)

    def get(self, *args, **kwargs):
        self.size.increment(-1)
        return super(Queue, self).get(*args, **kwargs)

    def qsize(self):
        """ Reliable implementation of multiprocessing.Queue.qsize() """
        return self.size.value

    def empty(self):
        """ Reliable implementation of multiprocessing.Queue.empty() """
        return not self.qsize()

opencv

Q:cv::Mat替换值

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
dst = src.clone();
cv::Mat Roi(dst, cv::Rect(x, y, width, height));
cv::RNG rnger(cv::getTickCount());
rnger.fill(Roi, cv::RNG::UNIFORM, cv::Scalar::all(0), cv::Scalar::all(256));

//创建图像
Mat img(Size(320,240), CV_8UC3);
//选择ROI
Mat roi(img, Rect(10, 10, 100, 100));
//将ROI的颜色填充为绿色,原始的img图像将会改变
roi = Scalar(0,255,0);

//setTo
cv::Mat image(8, 9, CV_8UC3);
image.setTo(255);  //将值全部设置成255
cv::Rect roi(1, 1, 4, 3);
image(roi).setTo(200);  //指定像素点区域的值都设置成200
image.at<cv::Vec3b>(0,0)[1] = 99;
image.setTo(88,image == 99);  //将M中等于99的值设置成88
//参数2是条件判断  可以是小于  大于等等

cv::Mat mask = cv::Mat::zeros(image.size(),CV_8UC3);
//建立与原图一样大小的mask图像，并将所有像素初始化为0
mask(roi).setTo(1);
mask.at<cv::Vec3b>(0,0)[1] = 66;
image.setTo(55, mask); //在矩阵M中，把矩阵mask中不为0的值全部变为55

Q:convertTo

1
2
3
4
5
6
void cv::Mat::convertTo(OutputArray m, int rtype, double alpha = 1, double beta = 0) const
image.convertTo(dst,CV_8UC1);//类型转换 CV_32FC1->CV_8UC1
image.convertTo(dst, CV_8CU1, 1.0/255, 0.5);
/*
alpha,beta-> image = (image->element * alpha)->element + beta
*/

Q:connectedComponentsWithStats

1
2
3
4
5
6
7
8
9
//像素相同的区域连通函数
//image:单通道
//4连通:上下左右,8连通:上下左右+左上、右上、右下、左下
cv::Mat labels,stats,centroids;
int n_comps = cv::connectedComponentsWithStats(image,labels,stats,centroids,4); 
//n_comps:连通区域的数量。
//labels:labels.shape = image.shape,每一个连通区域会有一个唯一标识,标识从0索引,0为背景
//stats:包含5个参数(x,y,h,w,s),对应每一个连通区域外接矩形的起始坐标x,y;外接矩形的width,height;s为labels对应的连通区域的像素个数
//centroids : 返回的是连通区域的质心

Q:cv2::threshold

1
2
3
4
5
6
7
8
9
10
11
12
13
cv::threshold (src,dst,thresh, maxval, type)
src:源矩阵,单通道
dst:目标
thresh:阈值,取值范围 0～255
maxval:填充色,取值范围 0～255
type：阈值类型

阈值	小于阈值的像素点	大于阈值的像素点
0	        置0	            置填充色
1	      置填充色	           置0
2	      保持原色	          置灰色
3	        置0	            保持原色
4	      保持原色	           置0

Q:读取摄像头/视频流

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
cap = cv2.VideoCapture("/home/jiaopan/Downloads/jd2.mp4")
    index = 0;
    detect = so.detectByBase64
    detect.restype = ctypes.c_char_p
    if cap.isOpened():
        while True:
            ret, frame = cap.read()
            #frame = cv2.resize(frame, (320, 320), interpolation=cv2.INTER_AREA)
            if index % 1 == 0:
                start = datetime.now()
                #cv2.imwrite("frame.jpg",frame)
                #result = detectFile(bytes("frame.jpg","utf-8"), ctypes.c_float(0.6))
                image = frame2base64(frame)
                result = detect(image, ctypes.c_float(0.4))
                end = datetime.now()
                print("time cost:",(end-start).total_seconds())
                result = ctypes.string_at(result, -1).decode("utf-8")
                result = json.loads(result)
                data = result["data"]
                print(data)
                for box in data:
                    x = int(box["location"]["x"])
                    y = int(box["location"]["y"])
                    width = int(box["location"]["width"])
                    height = int(box["location"]["height"])
                    cv2.putText(frame, box["label"], (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 255), 1)
                    cv2.rectangle(frame,(x,y),(x+width,y+height),(255,0,0),2)
            cv2.imshow('image', frame)
            k = cv2.waitKey(20)
            index += 1
            # q键退出
            if (k & 0xff == ord('q')):
                break

    cap.release()
    cv2.destroyAllWindows()

Q:图片转base64编码/base64编码还原图片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
def image_to_base64(image):
    image = cv2.imencode('.jpg', image)[1]
    image_code = str(base64.b64encode(image))[2:-1]
    return image_code
image = cv2.imread("test-04-output-0001.jpg")
print(image_to_base64(image))

def base64_to_image(base64_code):
    # base64解码
    img_data = base64.b64decode(base64_code)
    # 转换为np数组
    img_array = np.fromstring(img_data, np.uint8)
    # 转换成opencv可用格式
    img = cv2.imdecode(img_array, cv2.COLOR_RGB2BGR)
    return img
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
#include <iostream>
#include <fstream>
#include <sstream>
#include <map>
#include <opencv2/opencv.hpp>  

const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
bool isBase64( unsigned char c ){
    return (isalnum(c) || (c == '+') || (c == '/'));
}
std::string base64Decode(std::string const& encoded_string){
    int in_len = encoded_string.size();
    int i = 0;
    int j = 0;
    int index = 0;
    unsigned char char_array_4[4], char_array_3[3];
    std::string ret;

    while (in_len-- && (encoded_string[index] != '=') && isBase64(encoded_string[index])){
        char_array_4[i++] = encoded_string[index]; index++;
        if (i == 4){
            for (i = 0; i < 4; i++){
                char_array_4[i] = base64_chars.find(char_array_4[i]);
            }
            char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
            for (i = 0; (i < 3); i++){
                ret += char_array_3[i];
            }
            i = 0;
        }
    }

    if (i){
        for (j = i; j < 4; j++) char_array_4[j] = 0;
        for (j = 0; j < 4; j++) char_array_4[j] = base64_chars.find(char_array_4[j]);
        char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);
        char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
        for (j = 0; (j < i - 1); j++) ret += char_array_3[j];
    }

    return ret;
}
cv::Mat base64ToMat(std::string &base64_data){
    std::string decoded_string = base64Decode(base64_data);
    std::vector<uchar> data(decoded_string.begin(), decoded_string.end());
    cv::Mat image = imdecode(data, cv::IMREAD_UNCHANGED);
    return image;
}

Q：opencv Mat和byte互转

1
2
3
4
5
6
7
8
9
10
unsigned char* matToBytes(Mat image){
   int size = image.total() * image.elemSize();
	 unsigned char* bytes = new unsigned char[size];  // you will have to delete[] that later
	 std::memcpy(bytes, image.data, size * sizeof(unsigned char));
	 return bytes;
}
Mat bytesToMat(unsigned char* bytes,int width,int height){
    Mat image = Mat(height,width,CV_8UC3,bytes).clone(); // make a copy
    return image;
}

Q:opencv Mat 转std::vector std::vector转std::string

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
cv::Mat output = output.reshape(1, 1);
std::vector<double> vetor = (std::vector<double>)output;
std::stringstream ss;
ss << std::setprecision(16);//精度设置
std::copy(vector.begin(), vector.end(), std::ostream_iterator<double>(ss, ","));
std::string values = ss.str();
values.pop_back();//去掉末尾多余的分隔符
std::cout << values;

Mat convertToMat(std::string str) {
	std::vector<double> v;
	std::stringstream ss(str);
	ss << std::setprecision(16);
	std::string token;
	while (std::getline(ss, token, ',')) {
		v.push_back(std::stod(token));
	}
	Mat output = cv::Mat(v, true).reshape(1, 1);
	return output;
}

cv::FileStorage fs("../features.xml", cv::FileStorage::WRITE);
fs << "features" << output;
fs.release();

Q: opencv 截取图片的矩形区域/不规则图像

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#矩形
cv::rectangle(image, Rect(x, y, w, h), Scalar(0, 255, 0), 2);//绘制矩形框
Mat dst = image(cv::Range(y,y+h),cv::Range(x,x+w));

#不规则区域
image = cv2.imread("test.jpg")  
points = np.array([[[101, 116], [315, 18], [335, 64], [121, 161]]])  
height,width,_ = image.shape  
mask = np.zeros((height,width), np.uint8)  
cv2.polylines(mask, points, 1, 255)  
cv2.fillPoly(mask, points, 255)  
dst = cv2.bitwise_and(image, image, mask=mask)  
#设置白色背景  
# bg = np.ones_like(image, np.uint8) * 255  
# cv2.bitwise_not(bg, bg, mask=mask)  
# dst_white = bg + dst  
cv2.imwrite("roi.jpg", dst)

Q: Mat直接赋值

1
2
3
4
5
double left = 0,right = 0.5,top = 0,bottom = 0.5;
cv::Mat pbox = (cv::Mat_<double>(4,2)<<left,top,right,top,right,bottom,left,bottom); 

cv::Mat data_map(height, width, CV_8UC1, (unsigned char *)vec.data());
cv::Mat data_map(height, width, CV_32FC1, (float *)vec.data());

Q: Mat 行列 SUM/MAX/MIN/AVG

1
cv::reduce(pbox,sum_value,1,cv::REDUCE_SUM);

Q: 仿射变换

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
import cv2  
import numpy as np  
  
def perspective_transform_image(image, points):  
    rect_xmin, _, _, rect_xmax = np.sort(points[:, :, 0]).flatten()  
    rect_ymin, _, _, rect_ymax = np.sort(points[:, :, 1]).flatten()  
    image = image[rect_ymin:rect_ymax, rect_xmin:rect_xmax]  
  
    points[:, :, 0] = points[:, :, 0] - np.array(rect_xmin)  
    points[:, :, 1] = points[:, :, 1] - np.array(rect_ymin)  
    height, width, _ = image.shape  
    mask = np.zeros((height, width), np.uint8)  
    cv2.fillPoly(mask, points, (255, 255, 255))  
    dst = cv2.bitwise_and(image, image, mask=mask)  
    # 设置白色背景  
    # bg = np.ones_like(image, np.uint8) * 255    # cv2.bitwise_not(bg, bg, mask=mask)    # dst_white = bg + dst  
    name = str(time.time())  
    cv2.imwrite("../inference_results/%s_roi.jpg" % (name), dst)  
  
    left_top, right_top, right_bottom, left_bottom = np.squeeze(points, 0)  
    left_top_x, left_top_y = left_top  
    left_bottom_x, left_bottom_y = left_bottom  
    roi_height = int(  
        math.sqrt(math.pow(abs(left_bottom_x - left_top_x), 2) + math.pow(abs(left_bottom_y - left_top_y), 2)))  
    roi_width = int(math.sqrt(math.pow(height, 2) + math.pow(width, 2)))  
  
    source_points = np.float32([left_top, right_top, left_bottom, right_bottom])  
    target_points = np.float32([[0, 0], [roi_width, 0], [0, roi_height], [roi_width, roi_height], ])  
    mat = cv2.getPerspectiveTransform(source_points, target_points)  
    dst = cv2.warpPerspective(dst, mat, (roi_width, roi_height))  
    cv2.imwrite("../inference_results/%s_dst.jpg" % (name), dst)  
    return dst  
  
  
def affine_transform(image, angle):  
    height, width, _ = image.shape  
    center = (width // 2, height // 2)  
    matrx = cv2.getRotationMatrix2D(center, angle, 1.0)  
    rotated = cv2.warpAffine(image, matrx, (width, height), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)  
    cv2.imwrite("../inference_results/rotated.jpg", rotated)  
    x, y = center  
    print(x, y)  
    # xmin = 243  
    # ymin = 215    xmin = 100  
    ymin = 21  
    # points = np.array([[243, 236], [265, 215], [400, 348], [378, 369]])  
    points = np.array([[100, 119], [317, 21], [334, 60], [118, 159]])  
    points[:, 0] = points[:, 0] - xmin + 18  
    points[:, 1] = points[:, 1] - ymin  
  
    angle = math.radians(angle)  
  
    beta = round(math.sin(angle), 2)  
    alpha = round(math.cos(angle), 2)  
    print(beta, alpha)  
  
    x_points = points[:, 0].copy() - x  
    y_points = -(points[:, 1].copy() - y)  
    print("origin points:", x_points, y_points)  
  
    points[:, 0] = x_points * alpha - y_points * beta + x  
    print(points)  
    print(y_points * alpha + x_points * beta)  
    points[:, 1] = y - (y_points * alpha + x_points * beta)  
    print(points)  
  
    vis = rotated.copy()  
    cv2.drawContours(vis, [points], 0, (0, 0, 255), 2)  
    cv2.imwrite("../inference_results/vis.jpg", vis)

Java

Q：springboot切面编程实例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
@Aspect
@Component
@Slf4j
public class WebLogAspect {
	//定义一个切入点，对该包下的所有函数方法执行前，执行@Before()
    @Pointcut("execution(public * com.jp.controller..*.*(..))")
    public void webLog(){}

    @Before("webLog()")
    public void doBefore(JoinPoint joinPoint) throws Throwable {
        // 接收到请求，记录请求内容
        ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
        HttpServletRequest request = attributes.getRequest();

        // 记录下请求内容
        log.info("URL : " + request.getRequestURL().toString());
        log.info("HTTP_METHOD : " + request.getMethod());
        log.info("IP : " + request.getRemoteAddr());
        log.info("CLASS_METHOD : " + joinPoint.getSignature().getDeclaringTypeName() + "." + joinPoint.getSignature().getName());
        log.info("ARGS : " + Arrays.toString(joinPoint.getArgs()));

    }

    @AfterReturning(returning = "ret", pointcut = "webLog()")
    public void doAfterReturning(Object ret) throws Throwable {
        // 处理完请求，返回内容
        log.info("RESPONSE : " + ret);
    }

}

Q：Mac，MavenReportException: Error while creating archive: Unable to find javadoc command: The environment variable JAVA_HOME is not correctly set

在pom.xml文件中添加javadoc命令的位置

1
2
3
<properties>
 	<javadocExecutable>/Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/bin/javadoc</javadocExecutable>
</properties>

Q：maven引入本地依赖

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
<dependency>
    <groupId>com.oracle</groupId>
    <artifactId>ojdbc6</artifactId>
    <scope>system</scope>
    <version>1.0</version>
    <systemPath>${basedir}/lib/ojdbc6-11.2.0.1.0.jar</systemPath>
</dependency>

<build>
    <plugins>
    	<plugin>
    		<groupId>org.springframework.boot</groupId>
    		<artifactId>spring-boot-maven-plugin</artifactId>
    		<configuration>
    			<includeSystemScope>true</includeSystemScope>
    			<mainClass>mainclass</mainClass>
    		</configuration>
    	</plugin>
    </plugins>
</build>

Q：普通maven项目打包配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
<build>  
 <plugins>  
  <plugin>
     <artifactId>maven-compiler-plugin</artifactId>
     <version>2.3.2</version>
     <configuration>
         <source>1.8</source>
         <target>1.8</target>
     </configuration>
  </plugin>
  <plugin>
        <artifactId> maven-assembly-plugin</artifactId>
        <configuration>
             <descriptorRefs>
             <descriptorRef>jar-with-dependencies</descriptorRef>
             </descriptorRefs>
             <archive>
                  <manifest>
                       <mainClass>jiaopaner.spark.learning.App</mainClass>
                  </manifest>
             </archive>
        </configuration>
        <executions>
             <execution>
                  <id>make-assembly</id>
                  <phase>package</phase>
                  <goals>
                       <goal>single</goal>
                  </goals>
             </execution>
        </executions>
   </plugin>
 </plugins>  
</build>

Q:springboot-kafka集群

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# 显示启动
./bin/kafka-server-start.sh ./config/server.properties

# 在后台启动
./bin/kafka-server-start.sh -daemon ./config/server.properties

#创建topic
# --create：表示创建
# --zookeeper 后面的参数是zk的集群节点
# --replication-factor 3 ：表示复本数
# --partitions 3：表示分区数
# --topic test：表示topic的主题名称

./bin/kafka-topics.sh --create --zookeeper server01:2181,server02:2181,server03:2181 --replication-factor 3 --partitions 3 --topic test

#查看topic
./bin/kafka-topics.sh --list --zookeeper server01:2181
#删除topic
./bin/kafka-topics.sh --delete --zookeeper server01:2181 --topic test

#生产者命令
/bin/kafka-console-producer.sh --broker-list server01:9092 --topic test

#消费者命令
/bin/kafka-console-consumer.sh --zookeeper server01:2181 --from-beginning --topic test

Q:maven dependency scope设置

  • provided:编译和测试阶段依赖
1
2
3
4
5
6
<dependency>
    <groupId>org.jiaoper/groupId>
    <artifactId>utils-common</artifactId>
    <version>3.0-alpha-1</version>
    <scope>provided</scope>
</dependency>
  • test:测试阶段依赖
1
2
3
4
5
6
<dependency>
    <groupId>org.jiaoper/groupId>
    <artifactId>utils-common</artifactId>
    <version>3.0-alpha-1</version>
    <scope>test</scope>
</dependency>
  • compile:编译、测试和运行阶段依赖(默认)
1
2
3
4
5
6
<dependency>
    <groupId>org.jiaoper/groupId>
    <artifactId>utils-common</artifactId>
    <version>3.0-alpha-1</version>
    <scope>complie</scope>
</dependency>

StackOverflow

Q：利用nginx共享文件夹

nginx.conf添加如下配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
server {
    listen       8000;
    server_name  localhost;
    charset utf-8;
    location / {
	    root /home;#共享根目录下的home文件夹
	    # 索引
	    autoindex on;
	    # 显示文件大小
	    autoindex_exact_size on;
	    # 显示文件时间
	    autoindex_localtime on;
	    #index  index.html index.htm;
  }
}

Q:更改ubuntu源为阿里云源

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
sudo cp /etc/apt/sources.list /etc/apt/sources.list.bak

sudo vim /etc/apt/source.list

deb-src http://archive.ubuntu.com/ubuntu xenial main restricted #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-updates main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates universe
deb http://mirrors.aliyun.com/ubuntu/ xenial multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-updates multiverse
deb http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-backports main restricted universe multiverse #Added by software-properties
deb http://archive.canonical.com/ubuntu xenial partner
deb-src http://archive.canonical.com/ubuntu xenial partner
deb http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted
deb-src http://mirrors.aliyun.com/ubuntu/ xenial-security main restricted multiverse universe #Added by software-properties
deb http://mirrors.aliyun.com/ubuntu/ xenial-security universe
deb http://mirrors.aliyun.com/ubuntu/ xenial-security multiverse

apt-get update 

#or
mv /etc/apt/sources.list /etc/apt/sources.list.bak
echo "deb http://mirrors.163.com/debian/ jessie main non-free contrib" >> /etc/apt/sources.list
echo "deb http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list
echo "deb-src http://mirrors.163.com/debian/ jessie main non-free contrib" >>/etc/apt/sources.list
echo "deb-src http://mirrors.163.com/debian/ jessie-proposed-updates main non-free contrib" >>/etc/apt/sources.list

apt-get update

Q:oracle导入dmp备份文件

1
2
3
4
5
6
7
8
9
10
[root@centos ～]# su - oracle
[oracle@centos ～]# sqlplus 
请输入用户名:  sys as sysdba
输入口令:
SQL> CREATE USER 用户 IDENTIFIED BY 密码;
User created.

SQL> GRANT CONNECT, RESOURCE, DBA TO 用户;
Grant succeeded.
SQL> CONNECT 用户@orcl #新用户测试登陆

复制*.dmp文件到linux并加权

1
[root@centos ～]# chmod 777 *.dmp

导入dmp数据库备份文件

1
2
3
[root@centos ～] su - oracle
[oracle@centos ～]imp 用户名/密码@orcl file=/home/*.dmp ignore=y full=y;
 Import terminated successfully with warnings.(成功)

Q：scp文件/目录上传/下载

1
2
3
4
5
6
7
8
9
10
11
#本地上传到远程
#文件
scp /root/test.json root@远程服务器ip:/usr/local/src
#目录
scp -r /home/testdir root@远程服务器ip:/home/testdir

#远程下载到本地
#文件
scp root@远程服务器ip:/usr/local/src/test.json /root/
#目录
scp -r root@远程服务器ip:/home/testdir /home/

Q：Ubuntu增加swap交换空间

1
2
3
4
5
$ free -m #查看分区大小
$ dd if=/dev/zero of=/swapfile bs=1M count=3k #创建count*bs大小的swap文件
$ mkswap /swapfile #创建swap空间
$ swapon /swapfile #设置分区有效 ,swapoff 设置分区无效
$ vim /etc/fstab #开启启动：添加/swapfile swap swap defaults 0 0

Q：git创建分支

1
2
3
4
5
6
7
git clone -b <指定分支名> <远程仓库地址> #克隆指定分支
git branch -a  #查看分支
git branch <分支名> #创建分支
git checkout <分支名> #切换到分支
git add . #提交
git commit -m "" 
git push origin <分支名>

Q：git 子模块

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#添加
git submodule add <url> <path>
#更新
git submodule init
git submodule update

git submodule update --init --recursive

#删除
rm -rf 子模块源码目录
#删除子模块项
vi .gitmodules 
vi .git/config 
rm .git/module/子模块目录
git rm --cached 子模块名称

Q：glibc升级

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#http://ftp.gnu.org/gnu/glibc/
wget http://ftp.gnu.org/gnu/glibc/glibc-2.23.tar.gz
tar -zxvf glibc-2.23.tar.gz
cd glibc-2.23
mkdir build
cd build
../configure --prefix=/usr --disable-profile --enable-add-ons --with-headers=/usr/include --with-binutils=/usr/bin
make && make install 

'''
gawk: error while loading shared libraries: /lib64/libm.so.6: invalid ELF header
# 打开新终端执行如下命令
# cd /lib64
# unlink libm.so.6
# ln -s libm-2.23.so libm.so.6
# 回到原终端执行 make install
'''

Q：libstdc++升级

1
2
3
#下载最新版libstdc++
mv /usr/lib64/libstdc++.so.6  /usr/lib64/libstdc++.so.6.bak
ln -s /usr/lib64/libstdc++.so.6.0.26  /usr/lib64/libstdc++.so.6

Q：linux免密登录

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/worker/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/worker/.ssh/id_rsa
Your public key has been saved in /home/worker/.ssh/id_rsa.pub
The key fingerprint is:
SHA256:PmhwBGI0Gqgx8mCl5mHvj8ogyEfV9jnrzmLuKb2mOxQ worker@normalserver.inm.inner
The key's randomart image is:
+---[RSA 3072]----+
|o.*..            |
|*=.o ..          |
|=X   ..o         |
|= + .E. . .      |
| . o. o S+       |
|o o  + o  o      |
|+. o. + o.       |
|+ . o+ =+.       |
| o.. +X=++       |
+----[SHA256]-----+

#复制id_rsa.pub内容追加至目标连接主机.ssh/authorized_keys
$ ssh-copy-id -i ~/.ssh/id_rsa.pub worker@10.100.3.27
#手动方式
$ cd ~ 
$ mkdir .ssh && chmod 700 .ssh
$ cd .ssh
$ touch id_rsa.pub # 复制ssh-keygen主机的id_rsa.pub内容置此
$ cat id_rsa.pub >> authorized_keys
$ chmod 600 authorized_keys

Q：conda安装指定的cuda和cudnn

1
2
3
conda install pytorch==1.10.1 cudatoolkit=11.2 -c pytorch -c conda-forge

conda install tensorflow-gpu=1.4.1 cudatoolkit=8.0 cudnn=6.0 -c https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/linux-64/

Q：gcc版本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#centos7
devtoolset-3对应gcc4.x.x版本
devtoolset-4对应gcc5.x.x版本
devtoolset-6对应gcc6.x.x版本
devtoolset-7对应gcc7.x.x版本
devtoolset-8对应gcc8.x.x版本
devtoolset-9对应gcc9.x.x版本
devtoolset-10对应gcc10.x.x版本

yum install centos-release-scl
yum install devtoolset-7
#激活
scl enable devtoolset-7 bash
#永久激活
scl enable devtoolset-7 bash
sudo vim /etc/profile
source /opt/rh/devtoolset-7/enable
#centos8
dnf install gcc-toolset-10
scl enable gcc-toolset-10 bash

sudo yum remove centos-release-scl.noarch
sudo yum remove centos-release-scl-rh.noarch
sudo yum install centos-release-scl centos-release-scl-rh

CentOS-SCLo-scl.repo
[centos-sclo-sclo]
name=CentOS-7 - SCLo sclo
baseurl=https://mirrors.aliyun.com/centos/7/sclo/x86_64/sclo/
gpgcheck=0
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo

CentOS-SCLo-scl-rh.repo
[centos-sclo-rh]
name=CentOS-7 - SCLo rh
baseurl=https://mirrors.aliyun.com/centos/7/sclo/x86_64/rh/
gpgcheck=0
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo

1
2
3
4
5
6
7
8
9
10
11
12
13
14
#ubuntu
ls /usr/bin/gcc*
sudo apt-get install gcc-7 gcc-7
sudo apt-get install g++-9 g++-9
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 70  
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 70

sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90  
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90

sudo update-alternatives --config gcc
sudo update-alternatives --config g++
gcc -v
g++ -v

Q：sudo: /usr/lib/sudo/sudoers.so must be owned by uid 0

1
2
#ubuntu -> recovery mode  开机长按esc
$ pkexec chown root /usr/lib/sudo/sudoers.so

Q：evpp 安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
sudo yum -y install epel-release
sudo yum install glog-devel.x86_64
sudo yum install gflags-devel.x86_64
sudo yum install boost
sudo yum install boost-devel

sudo yum list | grep jsoncpp-devel
sudo yum install jsoncpp.x86_64
sudo yum install jsoncpp-devel.x86-64

#ubuntu
sudo apt-get install libgoogle-glog-dev
sudo apt-get install libgflags-dev
sudo apt-get install libboost-all-dev
sudo apt-get install libjsoncpp-dev

#################
#Optional:openssl libevent 
tar -zxvf openssl-1.1.1m.tar.gz
cd openssl-1.1.1m
./config --prefix=/usr/local/openssl
sudo make
sudo make install

tar -zxvf libevent-2.1.12-stable.tar.gz
cd libevent-2.1.12-stable
./configure --prefix=/usr/local && make && sudo make install
#################

git clone https://github.com/Qihoo360/evpp
cd evpp 
#if can not find openssl
vim CMakeLists.txt
#编辑CMakeLists.txt 链接openssl的lib和include
#include_directories("/usr/local/openssl/include")
#link_directories("/usr/local/openssl/lib")

cd evpp
git submodule update --init --recursive
mkdir -p build && cd build
cmake ..
make -j6

Q：centos时钟修复

1
2
yum install ntp -y
ntpdate ntp.aliyun.com

Q：grub引导修复

1
2
3
4
5
6
7
8
9
10
11
grub rescue> ls
grub rescue> ls (hd0,msdos1)/boot/grub

grub rescue> set root=(hd0,msdos1)
grub rescue> set prefix=(hd0,msdos1)/boot/grub

grub rescue> insmod normal
grub rescue> normal

$ sudo update-grub
$ sudo grub-install /dev/sda

Q：RuntimeError: Failed to export an ONNX attribute ‘onnx::Gather’, since it’s not constant, please try to make things (e.g., kernel size) static if possible

1
2
# output size 改为常量值
nn.AdaptiveAvgPool2d((None, 1)) -> nn.AdaptiveAvgPool2d((512, 1))

Q：pytoch numpy 打印值显示完全

1
2
torch.set_printoptions(threshold=np.inf,sci_mode=False,suppress=True,profile="full")
np.set_printoptions(threshold=np.inf,suppress=True)

Q：onnx convert to mnn

1
2
3
4
5
6
7
8
9
10
11
12
# MNN_BUILD_OPENCV=ON,MNN_BUILD_CONVERTER=ON
./MNN/schema/generate.sh
#mac/linux
cmake .. && make -j4 
#android
cd project/android
mkdir build_32 && cd build_32 && ../build_32.sh #armv7
mkdir build_64 && cd build_64 && ../build_64.sh #armv8

./MNNConvert -f ONNX --modelFile xx.onnx --MNNModel mnn_models/xx.mnn --bizCode biz --weightQuantBits 8

./MNNConvert -f TORCH --modelFile ./torchscript.pt --MNNModel mnn_models/test.mnn --bizCode biz

Q：Timeout waiting to lock execution history cache

1
find ~/.gradle -type f -name "*.lock" -delete

Q：kenlm train

1
2
3
4
5
6
7
8
9
10
11
12
yum install boost-devel boost-test boost
#ubuntu
sudo apt-get install libbz2-dev
sudo apt-get install libeigen3-dev
sudo apt-get install liblzma-dev

git clone https://github.com/kpu/kenlm.git
cd kenlm
mkdir build  
cd build
cmake ..   
make -j4
  • -o -> order执行命令的选项；
  • 3 -> N元语言模型(3-Gram)
  • —text(<) ./data.txt -> 语料路径；
  • —arpa(>) lm.arpa -> 保存语言模型文件名称
  • —prune -> 指定剪枝参数(0 4 4 4 -> 2-gram,3-gram,4-gram中频率小于4的剪枝)
  • -S 80% -> 申请内存
1
2
3
bin/lmplz -o 5 -S 80% --prune 2  <train_data.txt >models/text.arpa
bin/lmplz -o 3 --text ./data.txt  --arpa lm.arpa -S 80%
bin/lmplz -o 4 --prune 0 4 4 4 -S 40% --text train_char.txt --arpa ngram.pt

模型压缩

1
2
3
4
5
6
7
8
9
# -s 考虑起始符\<s>和结尾符<\s>
bin/build_binary lm.arpa lm.bin
bin/build_binary -s lm.arpa lm.bin

# 用于查看量化参数
bin/build_binary -s ngram.pt 

#根据上述结果选择合适参数量化
bin/build_binary trie -q 8 -b 8 -s ngram.pt Quantized_ngram.pt

lm.arpa

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
pro　　word　　back_pro
\1-grams:
-6.5514092  <unk> 0
0   <s>   -2.9842114
-1.8586434  </s>  0
-2.88382    !   -2.38764
-2.94351    world   -0.514311
-2.94351    hello   -0.514311
-6.09691    guys    -0.15553
 
\2-grams:
-3.91009    world ! -0.351469
-3.91257    hello world -0.24
-3.87582    hello guys  -0.0312
 
\3-grams:
-0.00108858 hello world !
-0.000271867    , hi hello !
 
\end\

1
2
3
'hello' -> return 1-grams pro 
'hello word' -> return 2-grams pro #find
'word hello' -> return back_pro(word)*pro(hello) #not find

demo

1
pip install pypi-kenlm

1
2
3
4
5
import kenlm
model = kenlm.LanguageModel("lm.bin")
sentences = ["滚滚长江东逝水", "长江滚滚水东逝", "长滚江水东滚逝"]
for sentence in sentences:
    print(sentence, model.score(sentence))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
#include "lm/model.hh"
#include <iostream>
#include <string>
int main() {
  using namespace lm::ngram;
  Model model("file.arpa");
  State state(model.BeginSentenceState()), out_state;
  const Vocabulary &vocab = model.GetVocabulary();
  std::string word;
  while (std::cin >> word) {
    std::cout << model.Score(state, vocab.Index(word), out_state) << '\n';
    state = out_state;
  }
}

Q：brew:sshfs has been disabled because it requires closed-source macFUSE

下载并安装:https://osxfuse.github.io/2022/04/08/macFUSE-4.2.5.html
sshfs: https://osxfuse.github.io/

1
2
3
4
5
6
brew formula ifuse
vim vim /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core/Formula/ifuse.rb

 # on_macos do
 #   disable! date: "2021-04-08", because: "requires closed-source macFUSE"
 # end

Q：mac 文件已损坏 无法打开

1
sudo xattr -r -d com.apple.quarantine /Applications/Sublime Text.app

Q：cmake指定安装目录

1
2
3
4
5
#方法一
cmake -DCMAKE_INSTALL_PREFIX=<安装路径>
#方法二
SET(CMAKE_INSTALL_PREFIX <install_path>)
install(TARGETS test DESTINATION <install_path>/bin)

Q：释放内存

1
sudo sh -c 'echo 1 > /proc/sys/vm/drop_caches'

Q：SquidMan设置代理

1
2
SquidMan 配置端口 添加client ip
client执行:all_proxy=代理服务器ip:端口

Q：进程绑定核心

1
2
cat /proc/cpuinfo | grep process
taskset -pc 0,2 11498

Q：supervisor

1
pip install supervisor

config.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
[program:barmak-nmt-server-0]
command=/data/servers/barmak_mt_server/build/NMTSERVER 8888 150 0 ../nmt_config.json
directory=/data/servers/barmak_mt_server/build
startsecs=0
stopwaitsecs=0
autostart=true
autorestart=true
stdout_logfile=/data/servers/barmak_mt_server/build/server_0.log
redirect_stderr=true
logfile_maxbytes=1024MB
stdout_logfile_backups=0

[program:barmak-nmt-server-1]
command=/data/servers/barmak_mt_server/build/NMTSERVER 8889 150 1 ../nmt_config.json
directory=/data/servers/barmak_mt_server/build
startsecs=0
stopwaitsecs=0
autostart=true
autorestart=true
stdout_logfile=/data/servers/barmak_mt_server/build/server_1.log
redirect_stderr=true
logfile_maxbytes=1024MB
stdout_logfile_backups=0

[inet_http_server]
port=*:9001
username=gks        
password=gks     

[supervisord]
logfile=./logs/supervisord.log
logfile_maxbytes=1024MB
logfile_backups=0
loglevel=info
redirect_stderr=true

启动

1
supervisord -c /path/config.ini #浏览器打开监控:ip:9001

Q：android bitmap像素值打印

1
2
3
4
5
6
7
8
9
10
11
12
private void showPixels(Bitmap bitmap){  
    for (int row = 0;row < bitmap.getHeight();row++){  
        for(int col = 0 ;col < bitmap.getWidth();col++){  
            int pixel = bitmap.getPixel(col,row);  
            Integer red = pixel >> 16 & 0xff;  
            Integer green = pixel >> 8 & 0xff;  
            Integer blue = pixel & 0xff;  
            String point = blue.toString() + "," + green.toString()+ "," +red.toString();  
            System.out.println(point);  
        }  
    }  
}

Q：android bitmap转RGB

1
2
3
4
5
6
7
8
9
10
11
12
13
14
public byte[] convertBitmapToRGBData(Bitmap bitmap){  
    int width = bitmap.getWidth();  
    int height = bitmap.getHeight();  
    int[] values = new int[width * height];  
    bitmap.getPixels(values,0,width,0,0,width,height);  
    byte[] rgb = new byte[width * height * 3];  
    for (int i = 0; i < values.length; i++) {  
        final int bit = values[i];  
        rgb[i * 3] = (byte) ((bit >> 16) & 0xFF);  
        rgb[i * 3 + 1] = (byte) ((bit >> 8) & 0xFF);  
        rgb[i * 3 + 2] = (byte) (bit & 0xFF);  
    }  
    return rgb;  
}

Q：jni byte[] 转unsigned char* *

1
2
3
4
5
6
7
8
unsigned char* bits = NULL;  
jbyte*  bytes = env->GetByteArrayElements(det_bitmap,0);  
int len = env->GetArrayLength(det_bitmap);  
bits = new unsigned char[len + 1];  
memset(bits,0,len + 1);  
memcpy(bits,bytes,len);  
bits[len] = 0;  
env->ReleaseByteArrayElements(det_bitmap,bytes,0);

Q：android项目打包

app/build.gradle

1
2
3
4
5
6
7
8
9
10
11
12
13
apply plugin: 'com.android.application' -> apply plugin: 'com.android.library'
注释 applicationId
android {
    ...
    task releaseJar(type: Copy) {
        delete 'build/libs/barmakocr.jar'
        from('build/intermediates/aar_main_jar/release')
        into('build/libs/')
        include('classes.jar')
        rename ('classes.jar','barmakocr.jar')
    }
}

app/src/main/AndroidManifest.xml

1
注释 <application>节点

Gradle->assemble 编译so,打包arr
Gradle->other->releaseJar 打包jar

引入jar包时,链接so库,添加如下配置:

1
2
3
4
5
sourceSets{
    main{
        jniLibs.srcDirs = ['libs']
    }
}

Q：cmake生成xcode工程报错

1
2
3
4
5
6
7
-- The C compiler identification is AppleClang 13.1.6.13160021
-- The CXX compiler identification is AppleClang 13.1.6.13160021
CMake Error at CMakeLists.txt:3 (project):
  No CMAKE_C_COMPILER could be found.

CMake Error at CMakeLists.txt:3 (project):
  No CMAKE_CXX_COMPILER could be found.
1
2
3
4
$ xcrun -find c++
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++
$ xcrun -find cc
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc
1
cmake -G Xcode .. -D CMAKE_C_COMPILER=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/cc -D CMAKE_CXX_COMPILER=/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++
1
2
sudo xcode-select -r
sudo xcode-select -s /Library/Developer/CommandLineTools

Q：ndk cmake生成的so过大

1
2
3
4
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Os -s")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Os -s")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions -frtti")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections -fdata-sections")

Q Android Cannot create service of type TaskExecuter using ProjectExecutionServices

1
2
./gradlew --stop
上述无法解决->重启电脑

Q 定位native崩溃

1
2
3
4
5
6
7
8
9
10
#编译-取消 SET(CMAKE_CXX_VISIBILITY_PRESET hidden)
#基于logcat定位
/andorid-sdk/platform-tools/adb logcat|andorid-sdk/ndk/21.3.6528147/ndk-stack -sym {path}/app/build/intermediates/cmake/release/obj/armeabi-v7a
#基于崩溃文件定位
andorid-sdk/ndk/21.3.6528147/ndk-stack -sym {path}/app/build/intermediates/cmake/release/obj/armeabi-v7a -dump crash.txt

开始解析 logcat 输出时查找第一行星号
*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
crash内容
...

Q Mac无法睡眠

1
2
3
4
5
6
7
pmset -g

System-wide power settings:
SleepDisabled 1  #睡眠被关闭
Currently in use:
lidwake 1
lowpowermode 0
1
sudo pmset -a disablesleep 0

Q cudnn 安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
tar -zxvf cudnn-10.2-linux-x64-v8.0.3.tgz
cd cuda    
sudo cp include/cudnn* /usr/local/cuda-11.3/include/  
sudo cp lib64/libcudnn* /usr/local/cuda-11.3/lib64  
sudo chmod a+r /usr/local/cuda-11.3/include/cudnn*.h /usr/local/cuda-11.3/lib64/libcudnn*

#查看cudnn版本
open | grep CUDNN_MAJOR -A 2


ll -h libcudnn*
sudo rm -rf libcudnn_adv_infer.so
sudo rm -rf libcudnn_adv_infer.so.8
sudo ln -sf libcudnn_adv_infer.so.8.9.4 libcudnn_adv_infer.so.8
sudo ln -sf libcudnn_adv_infer.so.8 libcudnn_adv_infer.so

sudo rm -rf libcudnn_adv_train.so
sudo rm -rf libcudnn_adv_train.so.8
sudo ln -sf libcudnn_adv_train.so.8.9.4 libcudnn_adv_train.so.8
sudo ln -sf libcudnn_adv_train.so.8 libcudnn_adv_train.so

sudo rm -rf libcudnn_cnn_infer.so
sudo rm -rf libcudnn_cnn_infer.so.8
sudo ln -sf libcudnn_cnn_infer.so.8.9.4 libcudnn_cnn_infer.so.8
sudo ln -sf libcudnn_cnn_infer.so.8 libcudnn_cnn_infer.so

sudo rm -rf libcudnn_cnn_train.so
sudo rm -rf libcudnn_cnn_train.so.8
sudo ln -sf libcudnn_cnn_train.so.8.9.4 libcudnn_cnn_train.so.8
sudo ln -sf libcudnn_cnn_train.so.8 libcudnn_cnn_train.so

sudo rm -rf libcudnn_ops_infer.so
sudo rm -rf libcudnn_ops_infer.so.8
sudo ln -sf libcudnn_ops_infer.so.8.9.4 libcudnn_ops_infer.so.8
sudo ln -sf libcudnn_ops_infer.so.8 libcudnn_ops_infer.so

sudo rm -rf libcudnn_ops_train.so
sudo rm -rf libcudnn_ops_train.so.8
sudo ln -sf libcudnn_ops_train.so.8.9.4 libcudnn_ops_train.so.8
sudo ln -sf libcudnn_ops_train.so.8 libcudnn_ops_train.so

sudo rm -rf libcudnn.so
sudo rm -rf libcudnn.so.8
sudo ln -sf libcudnn.so.8.9.4 libcudnn.so.8
sudo ln -sf libcudnn.so.8 libcudnn.so
ll libcudnn*

Q 查看linux 系统信息

1
uname -m && cat /etc/*release

Q 获取wiki语料

language: zh,en,ug,kk,… (ISO 639-1语言列表)
download: https://dumps.wikimedia.org/zhwiki

1
2
3
4
5
git clone https://github.com/attardi/wikiextractor 
cd wikiextractor 
python setup.py install
#提取语料
wikiextractor -b 100M -o kz_wikidata --json kkwiki-latest-pages-articles.xml

Q 指定GPU

1
2
export CUDA_VISIBLE_DEVICES=0
export CUDA_VISIBLE_DEVICES=0,1,2

Q pytorch 版本安装

1
2
3
4
5
# CUDA 11.1
pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html

# CUDA 10.2
pip install torch==1.10.1+cu102 torchvision==0.11.2+cu102 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html

Q onnx simplify

1
2
3
4
5
6
7
8
import onnx  
from onnxsim import simplify  
  
onnx_model = onnx.load("xx.onnx")  # load onnx model  
model_simp, check = simplify(onnx_model)  
assert check, "Simplified ONNX model could not be validated"  
onnx.save(model_simp, "xx_sim.onnx")  
print('finished exporting onnx')

Q cuda RuntimeError

1
2
3
RuntimeError: CUDA error: no kernel image is available for execution on the device
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

cuda版本与深度学习框架版本不匹配

1
2
3
4
5
# CUDA 11.1
pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html

# CUDA 10.2
pip install torch==1.10.1+cu102 torchvision==0.11.2+cu102 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html
1
2
3
4
#验证cuda可用
import torch 
torch.cuda.is_available() 
torch.zeros(8).cuda()

Q pip install uwsgi出错

1
conda install -c conda-forge uwsgi

**Q linux日志关键字查找和清空日志

1
2
3
4
5
6
7
#查找
grep 关键字 name.log
#清空
cat /dev/null > filename
: > filename
truncate -s 0 filename

Q NCCL install

1
2
3
4
5
6
7
8
#Network Installer for RedHat/CentOS 8
sudo dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo

#Network Installer for RedHat/CentOS 7
sudo yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo

#For RHEL/Centos: 
sudo yum install libnccl-2.8.4-1+cuda11.2 libnccl-devel-2.8.4-1+cuda11.2 libnccl-static-2.8.4-1+cuda11.2

Q JNI动态注册native方法

1
2
3
4
5
package com.test;  
  
public class TestRegister {  
    public static native String test();  
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
JavaVM* java_vm = nullptr;  
int32_t jni_version = 0;  
  
#define NATIVE_METHOD(class_name, function_name, native_function_name, signature) \  
{ #function_name, signature, (void*)(class_name ## _ ## native_function_name) }  
  
extern "C"  
JNIEXPORT jstring JNICALL  
Java_com_test_TestRegister_test(JNIEnv *env, jclass clazz) {  
    std::string text = "动态加载..";  
    return env->NewStringUTF(text.c_str());  
}  
  
bool jniRegisterNativeMethods(JNIEnv *env, const char *class_name, const JNINativeMethod *methods,  
                              int num_methods) {  
    jclass clazz = env->FindClass(class_name);  
    if (clazz == nullptr) {  
        return false;  
    }  
    return env->RegisterNatives(clazz, methods, num_methods) >= 0;  
}  
static JNINativeMethod methods[] = {  
        NATIVE_METHOD(Java_com_test_TestRegister, test, test, "()Ljava/lang/String;"),  
};  
  
  
jint JNI_OnLoad(JavaVM *vm, void * reserved) {  
    JNIEnv *env = nullptr;  
  
#ifdef JNI_VERSION_1_6  
    if (vm->GetEnv((void **) &env, JNI_VERSION_1_6) == JNI_OK) {  
        jni_version = JNI_VERSION_1_6;  
    }  
#endif  
  
#ifdef JNI_VERSION_1_4  
    if (jni_version == JNI_FALSE && vm->GetEnv((void **) &env, JNI_VERSION_1_4) == JNI_OK) {  
        jni_version = JNI_VERSION_1_4;  
    }  
#endif  
  
#ifdef JNI_VERSION_1_2  
    if (jni_version == JNI_FALSE && vm->GetEnv((void **) &env, JNI_VERSION_1_2) == JNI_OK) {  
        jni_version = JNI_VERSION_1_2;  
    }  
#endif  
  
    jniRegisterNativeMethods(env, "com/test/TestRegister", methods,  
                             sizeof(methods) / sizeof(methods[0]));  
  
    java_vm = vm;  
    return jni_version;  
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
static {  
    System.loadLibrary("test");  
}
Log.d("test", TestRegister.test());

//加载新的so
public static void copyFilesFromAssets(AssetManager assetManager, String assetsPath, String savePath){  
    try {  
        InputStream is = assetManager.open(assetsPath);  
        FileOutputStream fos = new FileOutputStream(new File(savePath + "/libtest.so"));  
        byte[] buffer = new byte[1024];  
        int byteCount = 0;  
        while ((byteCount = is.read(buffer)) != -1) {  
            fos.write(buffer, 0, byteCount);  
        }  
        fos.flush();  
        is.close();  
        fos.close();  
    } catch (Exception e) {  
        e.printStackTrace();  
    }  
}

String assets_lib_path = "4.1-lib/arm64-v8a/libtest.so";  
copyFilesFromAssets(getAssets(),assets_lib_path,getFilesDir().getPath());  
Log.d("TAG", getFilesDir().getPath());  
String fileNames[] = getFilesDir().list();  
for (String filename:fileNames){  
    Log.d("filename", filename);  
}  
System.load(getFilesDir().getPath() + "/libtest.so");

Q python传递图片数据(unsigned char*)至c++

1
2
3
4
image = cv2.imread("test.jpg")  
height,width,channels = image.shape
input_image = np.asarray(image,dtype=np.uint8)  
input_image = input_image.ctypes.data_as(c_char_p)

Q 查看安卓手机位数

1
adb shell getprop ro.product.cpu.abi

Q ffmpeg 安装

1
2
3
4
5
6
7
8
9
10
11
12
sudo yum install epel-release -y
sudo rpm --import http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
sudo rpm -Uvh http://li.nux.ro/download/nux/dextop/el7/x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm
sudo yum install ffmpeg ffmpeg-devel -y

sudo yum install x264-devel
git clone https://github.com/FFmpeg/FFmpeg.git
cd ffmpeg
mkdir build
./configure --enable-shared --enable-libx264 --enable-gpl --enable-openssl --enable-nonfree --disable-x86asm --prefix=./build
make -j6
make install
1
configuration: --prefix=/usr/local/Cellar/ffmpeg/4.2.2_4 --enable-shared --enable-pthreads --enable-version3 --enable-avresample --host-cflags=-fno-stack-check --host-ldflags= --enable-ffplay --enable-gnutls --enable-gpl --enable-libaom --enable-libbluray --enable-libmp3lame --enable-libopus --enable-librubberband --enable-libsnappy --enable-libsrt --enable-libtesseract --enable-libtheora --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxvid --enable-lzma --enable-libfontconfig --enable-libfreetype --enable-frei0r --enable-libass --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-librtmp --enable-libspeex --enable-libsoxr --enable-videotoolbox --disable-libjack --disable-indev=jack

Q 虚拟机配置静态ip 且宿主机互通 可访问外网
添加网卡
网络模式网络地址转换(NAT)->访问外网
仅主机(Host-Only)适配器(ifcfg-enp0s3)->宿主机互通
静态ip配置

1
2
cd /etc/sysconfig/network-scripts/
sudo vim ifcfg-enp0s3

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
IPADDR=192.168.56.3
NETMASK=255.255.255.0
GATEWAY=192.168.56.1
DNS1=114.114.114.114
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=enp0s3
UUID=e3fc06b7-3f5c-49fd-bb89-8a755f077384
DEVICE=enp0s3
ONBOOT=yes
1
service network restart

Q win11 wsl2 安装和迁移

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#控制面板->程序->启用或关闭windows功能->开启windows虚拟机监控程序平台、Linux 子系统（WSL2)、Hyper-V

bcdedit /set hypervisorlaunchtype auto #powershell

#appstore 下载安装ubuntu
#wsl2 linux内核更新包 https://wslstorestorage.blob.core.windows.net/wslblob/wsl_update_x64.msi

#迁移
wsl --shutdown
wsl --export Ubuntu-20.04 D:\export-wsl-Ubuntu-20.04
wsl --unregister Ubuntu-20.04
wsl --import Ubuntu-20.04 D:\ubuntu-20.04 D:\export-wsl-Ubuntu-20.04 --version 2
ubuntu2004.exe config --default-user [username]

#重启
wsl -l -v  
wsl -t Ubuntu  
wsl --shutdown  
wsl #启动

cat /etc/resolv.conf #win ip
ip a |grep "global eth0" #wsl ip

Q win11 wsl2 ssh

1
2
3
4
sudo apt remove openssh-server
sudo apt install openssh-server
sudo vim /etc/ssh/sshd_config
#export LD_LIBRARY_PATH=/usr/lib/wsl/lib:$LD_LIBRARY_PATH
1
2
3
4
5
6
Port 2222
#AddressFamily any
ListenAddress 0.0.0.0
#ListenAddress ::
PasswordAuthentication yes
PermitEmptyPasswords no
1
sudo service ssh --full-restart
1
2
3
#win11
ssh [wsl uername]@[wsl ip] -p 2222
ssh [wsl uername]@localhost -p 2222  #win11自动映射ip至wsl2
1
2
3
ssh uername@ip -p 2222
#WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED
#删除.ssh/known_hosts 对应ip的配置项

Q wsl挂载移动磁盘

1
2
sudo mount -t drvfs F: /mnt/f  #挂载
sudo umount /mnt/f #卸载磁盘

Q 查看端口当前连接数

1
netstat -ano|grep -E "8888"|grep "EST" |wc -l

Q kill相关进程

1
ps -ef |grep name |grep -v grep|awk '{print $2}'|xargs kill -9

Q github connection timeout

1
2
3
4
5
6
7
8
vim ~/.ssh/config

Host github.com
User git
Hostname ssh.github.com
PreferredAuthentications publickey
IdentityFile ~/.ssh/id_rsa
Port 443

Q cmake debug模式

1
2
3
4
5
6
7
# 设置 Debug 编译模式
set(CMAKE_BUILD_TYPE Debug)
# 添加调试相关的编译选项
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
# 添加生成调试信息的选项
set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} -g")

Q nginx负载均衡配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
upstream nmt_server {
	server 172.31.228.20:8888 weight=1;
	server 172.31.228.26:8888 weight=1;
	least_conn;
}
server {
	listen       8888;
	server_name  localhost;
	location /translateapi { 
		proxy_pass  http://nmt_server;
		proxy_set_header    Host    $http_host;
		proxy_set_header    X-Real-IP   $remote_addr;
		proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
	} 
}

Q nginx websocket ssl 配置

1
2
3
4
#编译安装
./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module
make -j6
make install
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
server {
	#listen 8082;
	listen       8082 ssl;
	server_name  *.cn;

	ssl_certificate      /root/apps/ssl/ssl_config/*.crt;
	ssl_certificate_key  /root/apps/ssl/ssl_config/*.key;

	ssl_session_cache    shared:SSL:1m;
	ssl_session_timeout  5m;

	ssl_ciphers  HIGH:!aNULL:!MD5;
	ssl_prefer_server_ciphers  on;

	location /asr {
		proxy_http_version 1.1;
		proxy_set_header Upgrade $http_upgrade;
		proxy_set_header Connection "Upgrade";
		proxy_pass http://127.0.0.1:8887;
		proxy_set_header Remote_addr $remote_addr;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	}
	location /ws {
		proxy_http_version 1.1;
		proxy_set_header Upgrade $http_upgrade;
		proxy_set_header Connection "Upgrade";
		proxy_pass http://127.0.0.1:8887;
		proxy_set_header Remote_addr $remote_addr;
		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	}
}

Q grpc protobuf 编译安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
#protobuf
sudo apt-get install libprotobuf-dev protobuf-compiler 
#源码编译
git clone https://github.com/protocolbuffers/protobuf.git 
cd protobuf 
git submodule update --init --recursive
mkdir build && cd build
cmake ..
#./configure --prefix=/usr/local/protobuf
make -j6
make install 

#grpc
apt-get update && apt-get install -y libssl-dev

git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc
cd grpc
git submodule update --init
#third-patry must install:abseil-cpp cares protobuf re2 zlib ssl

mkdir -p cmake/build
cd cmake/build
cmake \
  -DCMAKE_BUILD_TYPE=Release \
  -DgRPC_INSTALL=ON \
  -DgRPC_BUILD_TESTS=OFF \
  -DgRPC_CARES_PROVIDER=package \
  -DgRPC_ABSL_PROVIDER=package \
  -DgRPC_PROTOBUF_PROVIDER=package \
  -DgRPC_RE2_PROVIDER=package \
  -DgRPC_SSL_PROVIDER=package \
  -DgRPC_ZLIB_PROVIDER=package \
  ../..
sudo make j4 
sudo make install

protoc -I=. --grpc_out=. --plugin=protoc-gen-grpc=your_dir/grpc/cmake/build/grpc_cpp_plugin test.proto
protoc -I=$SRC_DIR --cpp_out=$OUT_DIR test.proto

Q Montreal-Forced-Aligner使用

Q git 修改历史提交

1
2
3
4
git rebase -i --root #or commit it
# pick -> edit
git commit --amend --reset-author --no-edit
git rebase --continue

**Q linux去除绿色背景色

1
eval "$(dircolors -p | sed 's/ 4[0-9];/ 01;/; s/;4[0-9];/;01;/g; s/;4[0-9] /;01 /' | dircolors /dev/stdin)"

**Q linux批量删除大量文件

1
find dir/* -name "*.npy" |xargs rm -r

**Q mac wifi l2tp

1
2
3
4
5
6
sudo vim /etc/ppp/ip-up

#!/bin/sh
/sbin/route add 192.168.211.1 -interface $1

sudo chmod +x /etc/ppp/ip-up

1
2
3
4
5
6
#vpn before
sudo sysctl net.link.generic.system.hwcksum_tx=0
sudo sysctl net.link.generic.system.hwcksum_rx=0
#after vpn
sudo sysctl net.link.generic.system.hwcksum_tx=1
sudo sysctl net.link.generic.system.hwcksum_rx=1

**Q linux 加密文件夹

1
2
3
4
5
6
7
8
9
sudo apt-get install encfs #ubuntu
sudo yum install encfs     #centos

mkdir /path/to/your/encrypted_folder 
#mount
encfs /path/to/your/encrypted_folder /path/to/your/mount_point 
cd /path/to/your/mount_point
#unmount
fusermount -u /path/to/your/mount_point

1
2
3
4
5
6
wget https://launchpad.net/veracrypt/trunk/1.26.14/+download/veracrypt-console-1.26.14-CentOS-7-x86_64.rpm

veracrypt --create /path/to/your/encrypted_container.vc

veracrypt /path/to/your/encrypted_container.vc /path/to/your/mount_point
veracrypt --dismount /path/to/your/mount_point

Q gcc 源码编译

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
sudo yum update -y
sudo yum groupinstall 'Development Tools' -y 
sudo yum install gmp-devel mpfr-devel libmpc-devel -y

wget https://ftp.gnu.org/gnu/gcc/gcc-9.3.0/gcc-9.3.0.tar.gz 
tar -xf gcc-9.3.0.tar.gz 
cd gcc-9.3.0

./contrib/download_prerequisites

mkdir build 
cd build
../configure --prefix=/usr/local --enable-bootstrap --enable-languages=c,c++,fortran,objc,obj-c++,java,ada --enable-shared --with-system-zlib --enable-threads=posix --enable-checking=release --with-demangler-in-ld --enable-multilib
make -j$(nproc)
sudo make install

sudo ln -sf /usr/local/bin/gcc /usr/bin/gcc 
sudo ln -sf /usr/local/bin/g++ /usr/bin/g++
gcc --version

Q vscode c++调试不显示变量值

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
wget https://ftp.gnu.org/gnu/gdb/gdb-15.2.tar.gz
tar -zxvf gdb-15.2.tar.gz
cd gdb-15.2
mkdir build && cd build
../configure --prefix=/usr/local --with-python=/usr/bin/python3
make -j6
make install
gdb --version

vim .gdbinit
'''
python
import sys
sys.path.insert(0, '/usr/local/gcc-9.4.0/libstdc++-v3/python')
from libstdcxx.v6.printers import register_libstdcxx_printers
register_libstdcxx_printers (None)
end
'''
(gdb) info pretty-printer
global pretty-printers:
  builtin
    mpx_bound128
  libstdc++-v6

Q 显卡安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#3090 >>>>>
yum install gcc
yum install gcc-c++
yum install kernel-devel-$(uname -r)
wget [https://cn.download.nvidia.com/XFree86/Linux-x86_64/535.161.07/NVIDIA-Linux-x86_64-535.161.07.run](https://cn.download.nvidia.com/XFree86/Linux-x86_64/535.161.07/NVIDIA-Linux-x86_64-535.161.07.run)
sh NVIDIA-Linux-x86_64-535.161.07.run

#A10 >>>>>>
yum install gcc
yum install gcc-c++
yum install kernel-devel-$(uname -r)
wget [https://cn.download.nvidia.com/tesla/525.147.05/NVIDIA-Linux-x86_64-525.147.05.run](https://cn.download.nvidia.com/tesla/525.147.05/NVIDIA-Linux-x86_64-525.147.05.run)
sh NVIDIA-Linux-x86_64-525.125.06.run

#ubuntu
sudo gedit /etc/modprobe.d/blacklist.conf
#blacklist nouveau 
#options nouveau modeset=0
sudo dpkg -i nvidia-driver-local-repo-ubuntu2204-560.35.03_1.0-1_amd64.deb 
sudo ubuntu-drivers devices
sudp apt-get install nvidia-driver-560
reboot

Q haproxy 安装和配置

1
2
3
4
5
6
wget https://www.haproxy.org/download/3.0/src/haproxy-3.0.0.tar.gz
tar -zxvf haproxy-3.0.0.tar.gz
cd haproxy-3.0.0
make TARGET=linux-glibc USE_OPENSSL=1 USE_PCRE=1 USE_ZLIB=1
sudo make install PREFIX=/usr/local
haproxy -v

  • 配置文件

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    vim /etc/haproxy/haproxy.cfg
    global
        log 127.0.0.1 local3 info
        log 127.0.0.1 local3 err
        chroot /var/lib/haproxy
        pidfile /var/run/haproxy.pid
        maxconn 500
        tune.bufsize 1048576
        tune.maxrewrite 16384
        stats socket /var/run/haproxy.sock mode 660 level admin expose-fd listeners
        daemon
        master-worker

    defaults
        log global
        mode http
        option httplog
        timeout connect 10s
        timeout client  30s
        timeout server  30s
        retries 3
        option redispatch

    listen stats
        bind *:8404
        stats enable
        stats uri /haproxy?stats
        stats refresh 10s
        stats admin if TRUE

    frontend ocr_frontend
        bind *:8881
        log global
        default_backend ocr_backend

    backend ocr_backend
        balance leastconn
        server ocr-0 172.21.16.17:8881 check maxconn 100 inter 2s
        server ocr-1 172.21.16.8:8881  check maxconn 100 inter 2s
        server ocr-2 172.21.16.7:8881  check maxconn 100 inter 2s

        option tcp-check

  • 日志配置

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    mkdir -p /var/log/haproxy
    touch /var/log/haproxy/haproxy.log
    touch /var/log/haproxy/error.log

    vim /etc/logrotate.d/haproxy
    /var/log/haproxy/*.log {
        size 1G
        copytruncate
        missingok
        notifempty
        nocreate
        sharedscripts
    }

    vim /etc/rsyslog.d/haproxy.conf
    $ModLoad imudp
    $UDPServerRun 514

    $ModLoad imtcp
    $InputTCPServerRun 514

    local3.info    /var/log/haproxy/haproxy.log
    local3.err    /var/log/haproxy/error.log
    & stop

    vim /etc/sysconfig/rsyslog
    SYSLOGD_OPTIONS="-c 2 -r -m 0"

    systemctl restart rsyslog

  • 启动

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    haproxy -f /etc/haproxy/haproxy.cfg

    vim /etc/systemd/system/haproxy.service
    [Unit]
    Description=HAProxy Load Balancer
    After=network.target

    [Service]
    Environment="CONFIG=/etc/haproxy/haproxy.cfg" "PIDFILE=/var/run/haproxy.pid"
    ExecStartPre=/usr/local/sbin/haproxy -f $CONFIG -c -q
    ExecStart=/usr/local/sbin/haproxy -Ws -f $CONFIG -p $PIDFILE
    ExecReload=/bin/kill -USR2 $MAINPID
    Restart=always
    KillMode=mixed

    [Install]
    WantedBy=multi-user.target

    sudo systemctl daemon-reload
    sudo systemctl start haproxy
    sudo systemctl enable haproxy
    sudo systemctl status haproxy

  • 重载配置

    1
    2
    haproxy -c -f /etc/haproxy/haproxy.cfg
    systemctl reload haproxy

Q conda环境中安装新版libstdc++

1
2
conda install -c conda-forge libstdcxx-ng
strings /root/miniconda3/envs/python3.8/lib/libstdc++.so.6 | grep GLIBCXX_3.4.30

Q docker install

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#ubuntu
sudo cp /etc/apt/sources.list /etc/apt/sources.list.bak

sudo sed -i "s@http://.*archive.ubuntu.com@http://mirrors.aliyun.com@g" /etc/apt/sources.list 

sudo sed -i "s@http://.*security.ubuntu.com@http://mirrors.aliyun.com@g" /etc/apt/sources.list


sudo apt remove docker docker-engine docker.io containerd runc -y 
sudo apt purge docker-ce docker-ce-cli containerd.io -y 
sudo rm -rf /var/lib/docker 
sudo rm -rf /var/lib/containerd

sudo apt install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg \
    lsb-release

curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

sudo apt update

sudo apt install -y docker-ce docker-ce-cli containerd.io

sudo systemctl enable docker
sudo systemctl start docker
sudo docker run --rm hello-world

#centos
sudo mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
sudo curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
sudo yum clean all
sudo yum makecache
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
sudo yum install -y docker-ce docker-ce-cli containerd.io
sudo docker version
sudo systemctl start docker

#option
yum list docker-ce --showduplicates | sort -r
yum install docker-ce-20.10.24 docker-ce-cli-20.10.24 containerd.io
yum list docker-compose-plugin --showduplicates | sort -r
yum install docker-compose-plugin-2.17.3-1.el

1
2
3
4
5
6
7
8
9
10
11
12
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": [
    "https://registry.cn-hangzhou.aliyuncs.com",
    "https://docker.mirrors.ustc.edu.cn",
    "https://hub-mirror.c.163.com",
    "https://docker.1ms.run"
  ],
  "insecure-registries": [],
  "debug": false
}
EOF
1
2
3
sudo systemctl daemon-reload 
sudo systemctl restart docker
sudo docker info

nvidia container toolkit

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
https://github.com/NVIDIA/nvidia-container-toolkit/releases
#ubuntu
tar -zxvf nvidia-container-toolkit_1.17.8_deb_amd64.tar.gz
cd release-v1.17.8-stable/packages/ubuntu18.04/amd64
sudo dpkg -i *.deb
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

cat /etc/docker/daemon.json
{
  "data-root": "/data/docker",
  "exec-opts": ["native.cgroupdriver=systemd"],
  "default-runtime": "nvidia",
  "registry-mirrors": ["https://docker.1ms.run"],
  "runtimes": {
    "nvidia": {
      "path": "nvidia-container-runtime",
      "runtimeArgs": []
    }
  }
}

Q NFS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#server
sudo apt install nfs-kernel-server -y

sudo vim /etc/exports
server_share_dir 192.168.1.0/24(rw,sync,no_subtree_check)

sudo exportfs -a
sudo systemctl restart nfs-kernel-server
showmount -e localhost

#client
sudo apt update 
sudo apt install nfs-common -y
sudo mkdir -p client_share_dir
sudo mount -t nfs -o resvport,rw server_ip:server_share_dir client_share_dir

Q github 大文件处理

1
2
3
4
5
6
git rm --cached big_file
#option
echo "big_file" >> .gitignore
git add .gitignore
git commit --amend -m "remove big file"
git push origin main --force
1
2
3
4
5
6
7
git lfs install
git rm --cached big_file
git lfs track "big_file"
git add big_file
git add .gitattributes
git commit --amend -m "move big file to Git LFS"
git push origin main --force

Q docker 目录迁移

1
2
3
4
5
6
7
8
9
10
11
docker system df
systemctl stop docker docker.service docker.socket
mkdir -p /data
rsync -avz /var/lib/docker /data/docker
sudo tee /etc/docker/daemon.json <<-'EOF' 
{ 
	"data-root": "/data/docker" 
} 
EOF

sudo systemctl start docker docker.service docker.socket

Q 远程访问sqlite

1
2
pip install datasette -i https://mirrors.aliyun.com/pypi/simple
datasette serve {path}/sqlite.db --port 8000 --host 0.0.0.0

Q NCCL通信问题排查

1
2
3
4
5
6
7
8
9
10
export NCCL_DEBUG=INFO
export NCCL_SOCKET_IFNAME=lo
export NCCL_IB_DISABLE=1 
export NCCL_P2P_DISABLE=1 
vllm serve ./Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 \
           --port 6888 --tensor-parallel-size 2 \
           --gpu-memory-utilization 0.92 \
           --max-model-len 32000 --block-size 16 \
           --swap-space 16 --max-num-seqs 4 \
           --enforce-eager --dtype auto

Q android 命令行工具

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
wget https://dl.google.com/android/repository/commandlinetools-linux-13114758_latest.zip
<SDK_ROOT> # SDK 根目录:~/android-sdk
└── cmdline-tools
	└── latest 
		├── bin
		├── lib
		└── ...
unzip commandlinetools-linux-13114758_latest.zip -d ~/android-sdk
mv ~/android-sdk/cmdline-tools/* ~/android-sdk/cmdline-tools/latest
#~/.bashrc
export ANDROID_HOME=~/android-sdk 
export PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/platform-tools

sdkmanager "platform-tools" "platforms;android-33" "build-tools;33.0.0"
sdkmanager "ndk;29.0.14206865"
#~/.bashrc
export ANDROID_NDK=$ANDROID_HOME/ndk/29.0.14206865

Q vscode 多版本共存

1
2
3
4
5
6
7
8
9
Windows x64 zip:
https://update.code.visualstudio.com/{version}/win32-x64-archive/stable
解压置vscode-{version},并新建data目录

macOS Intel chip:
https://update.code.visualstudio.com/{version}/darwin/stable
解压至vscode-{version}目录,并创建code-portable-data目录
cd vscode-{version}
xattr -dr com.apple.quarantine Visual\ Studio\ Code.app

Q 基于Docker 容器的开发环境

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
FROM ubuntu:22.04
ENV TZ=Asia/Shanghai
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends apt-utils && \
    apt-get install -y --no-install-recommends tzdata git python3 python-is-python3 gdb cmake vim build-essential pkg-config libssl-dev ca-certificates openssh-server python3-pip && \
    mkdir /var/run/sshd && echo 'root:klein' | chpasswd && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \
    apt-get clean && rm -rf /var/lib/apt/lists/*
RUN ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone
WORKDIR /app

EXPOSE 22

CMD ["/usr/sbin/sshd", "-D"]
1
docker run -d -p 2222:22 -v $(pwd)/path:/app/path --privileged --name name image_name:tag

Q huggingface 模型下载到本地

1
2
3
4
5
6
7
echo 'export HF_ENDPOINT=https://hf-mirror.com' >> ~/.bashrc
source ~/.bashrc

hf download tencent/Hunyuan-MT-7B --local-dir /data/resources/huggingface

#下载单个文件
hf download unsloth/DeepSeek-V4-Flash-0731-GGUF UD-IQ3_XXS/DeepSeek-V4-Flash-0731-UD-IQ3_XXS-00003-of-00004.gguf --local-dir DeepSeek-V4-Flash-0731-GGUF

Q v2raya 容器启动

1
2
3
4
5
6
7
8
9
10
11
12
13
14
docker run -d \
  --name v2raya \
  --restart always \
  -p 2017:2017 \
  -p 20170:20170 \
  -p 20171:20171 \
  -e V2RAYA_LOG_FILE=/tmp/v2raya.log \
  -e V2RAYA_V2RAY_BIN=/usr/bin/v2raya_core \
  -v /etc/v2raya:/etc/v2raya \
  mzz2017/v2raya

echo 'alias proxy_on="export http_proxy=http://192.168.211.166:20171 https_proxy=http://192.168.211.166:20171"' >> ~/.bashrc
echo 'alias proxy_off="unset http_proxy https_proxy"' >> ~/.bashrc
source ~/.bashrc

笔记
#小贴士
小贴士
https://jiaopaner.github.io/2020/04/03/小贴士/
作者
JiaoPan
发布于
2020年4月3日
许可协议