这是我的测试脚本
```python
import argparse
import sys
import torch
from sentence_transformers import SentenceTransformer
def main() -> int:
parser = argparse.ArgumentParser(description="Check text encoder output health")
parser.add_argument(
"--model",
type=str,
default="sentence-transformers/sentence-t5-xxl",
help="SentenceTransformer model path or model id",
)
parser.add_argument(
"--batch-size",
type=int,
default=2,
help="Batch size for encoding",
)
args = parser.parse_args()
texts = [
"hello world",
"test item title",
"Title: Toy Story; Brand: Pixar; Categories: Movies; Price: 19.99;",
]
model = SentenceTransformer(args.model)
emb = model.encode(
inputs=texts,
batch_size=args.batch_size,
show_progress_bar=False,
convert_to_tensor=True,
)
nan_count = torch.isnan(emb).sum().item()
inf_count = torch.isinf(emb).sum().item()
finite_ratio = torch.isfinite(emb).float().mean().item()
print("model:", args.model)
print("shape:", tuple(emb.shape))
print("dtype:", emb.dtype)
print("nan:", nan_count)
print("inf:", inf_count)
print("finite_ratio:", finite_ratio)
if finite_ratio < 1.0:
print("ERROR: encoder output contains non-finite values.")
return 1
print("OK: encoder output is fully finite.")
return 0
if __name__ == "__main__":
sys.exit(main())
```
结果全是NaN
