Skip to content

Base Knowledge

Install

docker start

# in python env
docker run -it --rm python:3.9.1-slim-buster

# in bash
docker run -it --rm python:3.9.1-slim-buster --bash

ubuntu/debian

  1. Prerequisties install

    sudo apt-get install build-essential checkinstall

sudo apt-get install libbz2-dev libc6-dev libgdbm-dev libncursesw5-dev libreadline-gplv2-dev libssl-dev libsqlite3-dev tk-dev

  2. Download and Untar

    wget -O /data/Python-3.10.14.tgz https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz
cd /data
tar -zxvf Python-3.10.14.tgz

  3. Install

    cd Python-3.10.14/
./configure --prefix=/opt/python/${PYTHON_VERSION}  --enable-optimizations

make
sudo make install

Python3.10

  4. create venv

    python3 -m venv /path/venv
source /path/venv/bin/activate

Commands

  • 防止终端缓存输出
nohup python -u cpu.py >cpu.out 2>&1 &

Snippets

  • thread 

    from threading import Thread

try:
    t = Thread(target=func, args=(request, results))
    t.start()
except:
    pass

  • server monitor

def get_cup_usage():
    """return float: 0.1-1..."""
    return psutil.cpu_percent()


def get_memory_usage():
    """获取内存可用大小 G"""
    return psutil.virtual_memory()._asdict()["available"] / (1024 ** 3)


def get_disk_status(path):
    """获取磁盘空间信息: return GB"""
    total, used, free = shutil.disk_usage(path)
    return total // (2 ** 30), used // (2 ** 30), free // (2 ** 30)


def get_disk_avaiable(path):
    """获取可用空间百分比: return 0.12434243"""
    total, used, free = get_disk_status(path)
    return free / total


def get_process_list_sorted_memory(n=5):
    '''
    Get list of running process sorted by Memory Usage
    '''
    objs = []
    # Iterate over the list
    for proc in psutil.process_iter():
        try:
            # Fetch process details as dict
            pinfo = proc.as_dict(attrs=['pid', 'name', 'username'])
            pinfo['rss'] = proc.memory_info().rss / (1024 * 1024)
            # Append dict to list
            objs.append(pinfo)
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass

    # Sort list of dict by key vms i.e. memory usage
    list_objs = sorted(objs, key=lambda x: x['rss'], reverse=True)

    return list_objs[:n]
  • json dumps
import datetime
import json
from json import JSONEncoder

ss = {
    "id": 456,
    "date": datetime.datetime.now()
}

# subclass JSONEncoder
class DateTimeEncoder(JSONEncoder):
    # Override the default method
    def default(self, obj):
        if isinstance(obj, (datetime.date, datetime.datetime)):
            # return obj.isoformat()
            return obj.strftime("%Y-%m-%dT%H:%M:%S")


print(DateTimeEncoder().encode(ss))
print(json.dumps(ss, indent=4, cls=DateTimeEncoder))

re

re.sub("^", "pre_/", "json")
# 'pre_/json'
  • singleton
class MetaClass(type):

    _instance ={}

    def __call__(cls, *args, **kwargs):

        """ Singelton Design Pattern  """

        if cls not in cls._instance:
            cls._instance[cls] = super(MetaClass, cls).__call__(*args, **kwargs)

        return cls._instance[cls]


class Test(metaclass=MetaClass):
    ...
  • decorator
def cfg_with_edb(name=None):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            edb = kwargs.get("edb")
            cfg = get_db_config_cache(edb)
            if name:
                cfg = cfg.get(name)
            return f(cfg, *args, **kwargs)
        return wrapper
    return decorator
  • tempfile
with tempfile.NamedTemporaryFile(mode="w+b", suffix='.py', delete=False) as f:
    url = f"{API_DB_URL}/csp/file/{rule_path}"
    resp_bytes = requests.get(url, timeout=10).content
    f.write(resp_bytes)
    spath = f.name
  • load-module
import importlib
spec = importlib.util.spec_from_file_location("entrypoint", fp)
foo = importlib.util.module_from_spec(spec)
spec.loader.exec_module(foo)
ret = foo.entrypoint(data)

pass-by-value-reference-and-assignment

https://mathspp.com/blog/pydonts/pass-by-value-reference-and-assignment

Some Python Rules

  1. string: length < 20, 缓存驻留
  2. hash(1)==hash(1.0)
  3. return error:
def some_func():
    try:
        return 'from_try'
    finally:
        return 'from_finally'

>>> some_func()
'from_finally'
  1. id/is/class create

    当我们连续两次进行这个操作时, Python会将相同的内存地址分配给第二个对象. 因为 (在CPython中) id 函数使用对象的内存地址作为对象的id值, 所以两个对象的id值是相同的.

# python3.9.10
class WTF(object):
  def __init__(self): print("I")
  def __del__(self): print("D")

>>> WTF() is WTF()
I
I
D
D
False
>>> id(WTF()) == id(WTF())
I
D
I
D
True
  1. for: exprlist
for i in range(4):
    print(i)
    i = 10

# 0
# 1
# 2
# 3
  1. Evaluation time discrepancy

    in在声明时执行，条件在运行时执行。 切片赋值在原来对象上操作

array = [1, 8, 15]
g = (x for x in array if array.count(x) > 0)
array = [2, 8, 22]
# >>> print(list(g))
# [8]
array_1 = [1,2,3,4]
g1 = (x for x in array_1)
array_1 = [1,2,3,4,5]
# print(list(g1))
# [1, 2, 3, 4]
array_2 = [1,2,3,4]
g2 = (x for x in array_2)
array_2[:] = [1,2,3,4,5]
# print(list(g2))
# [1, 2, 3, 4, 5]