Best Practice VR Multiplayer?

Hello Forum!

I've tried for quite a while to get a properly functioning VR setup, but are running into some problems.
I managed to set up a room and track the headset rotation + position, but struggeling with the position of the controllers.
I might have done this completely wrong, and it feels like i've done this completely inefficient, so maybe someone knows a better way to do this!
What i have:

For the Body

public class NetworkedBody : Photon.MonoBehaviour {

    public GameObject avatar;
    public Transform playerGlobal;
    public Transform playerLocal;

    void Start() {
        if (photonView.isMine) {
            playerGlobal = GameObject.Find("OVRPlayerController").transform;
            playerLocal = playerGlobal.Find("OVRCameraRig/TrackingSpace/CenterEyeAnchor");
            avatar.transform.SetParent(playerLocal);
            avatar.transform.localPosition = Vector3.zero;
            // avatar.SetActive(false);
        }
    }

    void OnPhotonSerializeView(PhotonStream stream, PhotonMessageInfo info){
        if (stream.isWriting){
            stream.SendNext(playerGlobal.position);
            stream.SendNext(playerGlobal.rotation);
            stream.SendNext(playerLocal.localPosition);
            stream.SendNext(playerLocal.localRotation);
        }
        else {
            transform.position = (Vector3)stream.ReceiveNext();
            transform.rotation = (Quaternion)stream.ReceiveNext();
            avatar.transform.localPosition = (Vector3)stream.ReceiveNext();
            avatar.transform.localRotation = (Quaternion)stream.ReceiveNext();
        }
    }
}
For the Left Glove
public class NetworkedLeftGlove : Photon.MonoBehaviour {

    public GameObject leftGlove;
    public Transform playerGlobal;
    public Transform playerLocal;

    void Start(){
        if (photonView.isMine){
            playerGlobal = GameObject.Find("OVRPlayerController").transform;
            playerLocal = playerGlobal.Find("OVRCameraRig/TrackingSpace/LeftHandAnchor");
            leftGlove.transform.SetParent(playerLocal);
            leftGlove.transform.localPosition = Vector3.zero;
            // avatar.SetActive(false);
        }
    }

    void OnPhotonSerializeView(PhotonStream stream, PhotonMessageInfo info){
        if (stream.isWriting){
            stream.SendNext(playerGlobal.position);
            stream.SendNext(playerGlobal.rotation);
            stream.SendNext(playerLocal.localPosition);
            stream.SendNext(playerLocal.localRotation);
        }
        else {
            transform.position = (Vector3)stream.ReceiveNext();
            transform.rotation = (Quaternion)stream.ReceiveNext();
            leftGlove.transform.localPosition = (Vector3)stream.ReceiveNext();
            leftGlove.transform.localRotation = (Quaternion)stream.ReceiveNext();
        }
    }
}
And basically the same for the right glove.
The gloves seem to move only a little bit of the distance needed.
What is a better aproach for this?