Coding01

...
def tokenize(self, message: Message, attribute: Text) -> List[Token]:
    import jieba

    text = message.get(attribute)

    tokenized = jieba.tokenize(text)
    print('******')
    print(f"{[t for t in tokenized]}")
    print('******')
    
    tokens = [Token(word, start) for (word, start, end) in tokenized]
    return self._apply_token_pattern(tokens)
...

language: zh

pipeline:
  - name: components.fanlyJiebaTokenizer.JiebaTokenizer
  - name: CRFEntityExtractor
  - name: CountVectorsFeaturizer
    OOV_token: oov
    token_pattern: '(?u)\b\w+\b'
  - name: KeywordIntentClassifier

NLU model loaded. Type a message and press enter to parse it.
Next message:
我想找地方吃饭
******
Building prefix dict from the default dictionary ...
Loading model from cache /var/folders/cz/kq5sssg12jx887hj62hwczrr0000gn/T/jieba.cache
Loading model cost 0.729 seconds.
Prefix dict has been built successfully.
[('我', 0, 1), ('想', 1, 2), ('找', 2, 3), ('地方', 3, 5), ('吃饭', 5, 7)]
******
{
  "text": "我想找地方吃饭",
  "intent": {
    "name": "eat_search",
    "confidence": 1.0
  },
  "entities": []
}
Next message:

/rasa/nlu/tokenizers

import glob
import logging
import os
import shutil
import typing
from typing import Any, Dict, List, Optional, Text

from rasa.nlu.components import Component
from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
from rasa.shared.nlu.training_data.message import Message

logger = logging.getLogger(__name__)


if typing.TYPE_CHECKING:
    from rasa.nlu.model import Metadata


class TensorFlowTextTokenizer(Tokenizer):
    """This tokenizer is a wrapper for tensorflow_text (https://www.tensorflow.org/tutorials/tensorflow_text/intro)."""

    supported_language_list = ["zh"]

    defaults = {
        "model_handle": "https://hub.tensorflow.google.cn/google/zh_segmentation/1",
        # Flag to check whether to split intents
        "intent_tokenization_flag": False,
        # Symbol on which intent should be split
        "intent_split_symbol": "_",
        # Regular expression to detect tokens
        "token_pattern": None,
    }  # default don't load custom dictionary

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
        """Construct a new intent classifier using the TensorFlow framework."""

        super().__init__(component_config)

    @classmethod
    def required_packages(cls) -> List[Text]:
        return ["tensorflow", "tensorflow_text"]

    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
        import tensorflow_text as tftext
        import tensorflow as tf

        # 设定模型的 UR
        self.model_handle = self.component_config.get("model_handle")
        segmenter = tftext.HubModuleTokenizer(self.model_handle)

        text = message.get(attribute)
        print(text)
        tokens, starts, ends = segmenter.tokenize_with_offsets(text)
        tokens_list = tokens.numpy()
        starts_list = starts.numpy()
        print('******')
        print(f"{[t.decode('utf-8') for t in tokens_list]}")
        print(f"{[t for t in starts_list]}")
        print('******')
        
        tokensData = [Token(tokens_list[i], starts_list[i]) for i in range(len(tokens_list))]
        return self._apply_token_pattern(tokensData)

from rasa.nlu.tokenizers.tensorflow_text_tokenizer import TensorFlowTextTokenizer

...

component_classes = [
    # utils
    SpacyNLP,
    MitieNLP,
    HFTransformersNLP,
    # tokenizers
    MitieTokenizer,
    SpacyTokenizer,
    WhitespaceTokenizer,
    ConveRTTokenizer,
    JiebaTokenizer,
    TensorFlowTextTokenizer,
    ...
]

poetry run rasa init

nlu:
- intent: eat_search
  examples: |
    - 我想找地方吃饭
    - 我想吃[火锅](food)了
    - 找个吃[拉面](food)的地方
    - 附近有什么好吃的地方吗？

language: zh

pipeline:
  - name: TensorFlowTextTokenizer
  - name: CRFEntityExtractor
  - name: CountVectorsFeaturizer
    OOV_token: oov
    token_pattern: '(?u)\b\w+\b'
  - name: KeywordIntentClassifier

// 训练
poetry run rasa train nlu

import TensorFlow
import TrainingLoop

var x: Float = 0
let η: Float = 0.01
let maxIterations = 100

@differentiable
func f(_ x: Float) -> Float {
    return 5 * pow(x, 3) + 2 * pow(x, 2) - 3 * x
}

print("Before optimization, ", terminator: "")
print("x: \(x) and f(x): \(f(x))")

// Optimization loop
for _ in 1...maxIterations {
    /// Derivative of `f` w.r.t. `x`.
    let 𝛁xF = gradient(at: x) { x -> Float in
        return f(x)
    }
    // Optimization step: update `x` to maximize `f`
    x += η * 𝛁xF
}

print("After gradient ascent, ", terminator: "")
print("input: \(x) and output: \(f(x))")

for _ in 1...maxIterations {
    let 𝛁xF = gradient(at: x) { x -> Float in
        return f(x)
    }
    // Optimization step: update `x` to maximize `f`
    x.move(along: 𝛁xF.scaled(by: -η))
}
print("After gradient descent, ", terminator: "")
print("input: \(x) and output: \(f(x))")

@inlinable public func gradient<T, R>(at x: T, in f: @differentiable (T) -> Tensor<R>) -> T.TangentVector where T : Differentiable, R : TensorFlowFloatingPoint

pip install "tensorflow_text>=2.4.0b0"

import tensorflow_text as text
import tensorflow as tf

# 设定模型的 UR
MODEL_HANDLE = "https://hub.tensorflow.google.cn/google/zh_segmentation/1"
segmenter = text.HubModuleTokenizer(MODEL_HANDLE)

# 分割[新华社北京]。
input_text = ["新华社北京"]
tokens, starts, ends = segmenter.tokenize_with_offsets(input_text)

print(tokens.to_list())

first = tokens.to_list()[0][0]
second = tokens.to_list()[0][1]

print(first.decode('utf-8'))
print(second.decode('utf-8'))

pip install -q tensorflow-recommenders
pip install -q --upgrade tensorflow-datasets

import os
import pprint
import tempfile

from typing import Dict, Text

import numpy as np
import tensorflow as tf
import tensorflow_datasets as tfds
import tensorflow_recommenders as tfrs
import ssl

os.environ['HTTP_PROXY'] = 'http://0.0.0.0:8888'
os.environ['HTTPS_PROXY'] = 'http://0.0.0.0:8888'
ssl._create_default_https_context =  ssl._create_unverified_context

# Ratings data.
ratings = tfds.load('movielens/100k-ratings', split="train", data_dir = os.path.join(os.getcwd(), "data"))
# Features of all the available movies.
movies = tfds.load('movielens/100k-movies', split="train", data_dir = os.path.join(os.getcwd(), "data"))

import PythonKit
print(Python.version)

import TensorFlow
import PythonKit
import Foundation

let os = Python.import("os")
let np = Python.import("numpy")
let metrics = Python.import("sklearn.metrics")

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data(
    path = os.path.join(os.getcwd(), "data/mnist.npz")
)

if !Bool(os.path.exists("mnist.npz"))! {
    os.popen("wget https://s3.amazonaws.com/img-datasets/mnist.npz").read()
}

let mnist = np.load("mnist.npz")

import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras import layers, models
import tensorflow_datasets as tfds
import os
import ssl

os.environ['HTTP_PROXY'] = 'http://0.0.0.0:8888'
os.environ['HTTPS_PROXY'] = 'http://0.0.0.0:8888'
ssl._create_default_https_context =  ssl._create_unverified_context

(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()

$ export PATH=/Library/Developer/Toolchains/swift-latest/usr/bin:"${PATH}"

class StepsInteractor {
    let healthStore = HKHealthStore()
    let stepCountType = HKObjectType.quantityType(forIdentifier: HKQuantityTypeIdentifier.stepCount)!
    // Access Step Count
    let healthKitTypes: Set = [ HKObjectType.quantityType(forIdentifier: HKQuantityTypeIdentifier.stepCount)! ]
    
    func retrieveStepsWithAuth(completion: @escaping (Double) -> Void) {
        // Check for Authorization
        if (healthStore.authorizationStatus(for: stepCountType) != HKAuthorizationStatus.sharingAuthorized) {
            healthStore.requestAuthorization(toShare: healthKitTypes, read: healthKitTypes) { (success, error) in
                if (success) {
                    // Authorization Successful
                    self.getSteps { (result) in
                        completion(result)
                    }
                } else {
                    completion(-1)
                }
            }
        } else {
            self.getSteps { (result) in
                completion(result)
            }
        }
    }
    
    func getSteps(completion: @escaping (Double) -> Void) {
        let stepsQuantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!

        let now = Date()
        let startOfDay = now - 2.days
        var interval = DateComponents()
        interval.day = 1
        
        let query = HKStatisticsCollectionQuery(
             quantityType: stepsQuantityType,
             quantitySamplePredicate: nil,
             options: [.cumulativeSum],
             anchorDate: startOfDay,
             intervalComponents: interval)
        
        query.initialResultsHandler = { _, result, error in
                var resultCount = 0.0
                result!.enumerateStatistics(from: startOfDay, to: now) { statistics, _ in
                if let sum = statistics.sumQuantity() {
                    // Get steps (they are of double type)
                    resultCount = sum.doubleValue(for: HKUnit.count())
                } // end if
                // Return
                completion(resultCount)
            }
        }
        
        query.statisticsUpdateHandler = {
            query, statistics, statisticsCollection, error in
            // If new statistics are available
            if let sum = statistics?.sumQuantity() {
                let resultCount = sum.doubleValue(for: HKUnit.count())
                // Return
                completion(resultCount)
            } // end if
        }
        
        healthStore.execute(query)
    }
}